<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  

   <!-- 百度联盟 -->
   <meta name="baidu_union_verify" content="46e227ad81d5513c6cd8288c18c6ebad">

   <!-- google analytics -->
   <meta name="google-site-verification" content="SzCGQVmA8Mtk40elee-bCGpq2YSCAmEulSNEZHYCkFc" />

   <!--百度站长之家验证-->
   <meta name="baidu-site-verification" content="4woRik4rfk" />

   <meta name="description" content="坑要一个个填，路要一步步走！—— from zhisheng的博客">
   <meta name="keywords" content="Java,架构,后端,服务端,RocketMQ,分布式消息队列,分布式存储,技术博客,HBase,ElasticSearch,Spring,Spring Boot,Spring Boot 2.0,Spring Cloud,Spring MVC,Java EE,前端,HTML,MyBatis,Android,Docker,Mac,Consul,Kafka,Logstash,Kibana,MySQL,Maven,Nginx,Python,RabbitMQ,ActiveMQ,JVM">


  <title>Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法 | zhisheng的博客</title>
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta name="description" content="介绍：ElasticSearch 是一个基于 Lucene 的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎，基于 RESTful web 接口。Elasticsearch 是用 Java 开发的，并作为Apache许可条款下的开放源码发布，是当前流行的企业级搜索引擎。设计用于云计算中，能够达到实时搜索，稳定，可靠，快速，安装使用方便。 Elasticsearch中，内置了很多分词器（an">
<meta name="keywords" content="ElasticSearch">
<meta property="og:type" content="article">
<meta property="og:title" content="Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法">
<meta property="og:url" content="http://www.54tianzhisheng.cn/2017/09/08/Elasticsearch-analyzers/index.html">
<meta property="og:site_name" content="zhisheng的博客">
<meta property="og:description" content="介绍：ElasticSearch 是一个基于 Lucene 的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎，基于 RESTful web 接口。Elasticsearch 是用 Java 开发的，并作为Apache许可条款下的开放源码发布，是当前流行的企业级搜索引擎。设计用于云计算中，能够达到实时搜索，稳定，可靠，快速，安装使用方便。 Elasticsearch中，内置了很多分词器（an">
<meta property="og:locale" content="zh-Hans">
<meta property="og:image" content="https://ws3.sinaimg.cn/large/006tNc79gy1fp3jkmizmpj30o00didgn.jpg">
<meta property="og:updated_time" content="2018-03-24T02:25:43.000Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法">
<meta name="twitter:description" content="介绍：ElasticSearch 是一个基于 Lucene 的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎，基于 RESTful web 接口。Elasticsearch 是用 Java 开发的，并作为Apache许可条款下的开放源码发布，是当前流行的企业级搜索引擎。设计用于云计算中，能够达到实时搜索，稳定，可靠，快速，安装使用方便。 Elasticsearch中，内置了很多分词器（an">
<meta name="twitter:image" content="https://ws3.sinaimg.cn/large/006tNc79gy1fp3jkmizmpj30o00didgn.jpg">
  
    <link rel="alternative" href="/atom.xml" title="zhisheng的博客" type="application/atom+xml">
  
  
    <link rel="icon" href="/img/favicon.ico">
  
  
      <link rel="stylesheet" href="//cdn.bootcss.com/animate.css/3.5.0/animate.min.css">
  
  <link rel="stylesheet" href="/css/style.css">
  <link rel="stylesheet" href="/font-awesome/css/font-awesome.min.css">
  <link rel="apple-touch-icon" href="/apple-touch-icon.png">
  
  
      <link rel="stylesheet" href="/fancybox/jquery.fancybox.css">
  
  <!-- 加载特效 -->
  <script src="/js/pace.js"></script>
  <link href="/css/pace/pace-theme-flash.css" rel="stylesheet" />
  <script>
      var yiliaConfig = {
          rootUrl: '/',
          fancybox: true,
          animate: true,
          isHome: false,
          isPost: true,
          isArchive: false,
          isTag: false,
          isCategory: false,
          open_in_new: true
      }
  </script>
</head>
<body>
  <div id="container">
    <div class="left-col">
    <div class="overlay"></div>
<div class="intrude-less">
    <header id="header" class="inner">
        <a href="/" class="profilepic">
            
            <img lazy-src="/img/avatar.png" class="js-avatar">
            
        </a>

        <hgroup>
          <h1 class="header-author"><a href="/">zhisheng</a></h1>
        </hgroup>

        
        <p class="header-subtitle">坑要一个个填，路要一步步走！</p>
        
        
        
            <div id="switch-btn" class="switch-btn">
                <div class="icon">
                    <div class="icon-ctn">
                        <div class="icon-wrap icon-house" data-idx="0">
                            <div class="birdhouse"></div>
                            <div class="birdhouse_holes"></div>
                        </div>
                        <div class="icon-wrap icon-ribbon hide" data-idx="1">
                            <div class="ribbon"></div>
                        </div>
                        
                        <div class="icon-wrap icon-link hide" data-idx="2">
                            <div class="loopback_l"></div>
                            <div class="loopback_r"></div>
                        </div>
                        
                        
                        <div class="icon-wrap icon-me hide" data-idx="3">
                            <div class="user"></div>
                            <div class="shoulder"></div>
                        </div>
                        
                    </div>
                    
                </div>
                <div class="tips-box hide">
                    <div class="tips-arrow"></div>
                    <ul class="tips-inner">
                        <li>菜单</li>
                        <li>标签</li>
                        
                        <li>友情链接</li>
                        
                        
                        <li>关于我</li>
                        
                    </ul>
                </div>
            </div>
        

        <div id="switch-area" class="switch-area">
            <div class="switch-wrap">
                <section class="switch-part switch-part1">
                    <nav class="header-menu">
                        <ul>
                        
                            <li><a href="/">主页</a></li>
                        
                            <li><a href="/tags/Flink/">Flink</a></li>
                        
                            <li><a href="/tags/ElasticSearch/">ElasticSearch</a></li>
                        
                            <li><a href="/tags/SpringBoot/">Spring Boot</a></li>
                        
                        </ul>
                    </nav>
                    <nav class="header-nav">
                        <ul class="social">
                            
                                <a class="fl wechat" target="_blank" href="wechat:zhisheng_tian" title="wechat">wechat</a>
                            
                                <a class="fl mail" target="_blank" href="mailto://zhisheng2018@gmail.com" title="mail">mail</a>
                            
                                <a class="fl github" target="_blank" href="https://github.com/zhisheng17/" title="github">github</a>
                            
                                <a class="fl zhihu" target="_blank" href="https://www.zhihu.com/people/tian-zhisheng/activities" title="zhihu">zhihu</a>
                            
                        </ul>
                    </nav>
                </section>
                
                
                <section class="switch-part switch-part2">
                    <div class="widget tagcloud" id="js-tagcloud">
                        <a href="/tags/AJAX/" style="font-size: 10px;">AJAX</a> <a href="/tags/ActiveMQ/" style="font-size: 10px;">ActiveMQ</a> <a href="/tags/Android/" style="font-size: 10px;">Android</a> <a href="/tags/Bootstrap/" style="font-size: 13px;">Bootstrap</a> <a href="/tags/Consul/" style="font-size: 10px;">Consul</a> <a href="/tags/Docker/" style="font-size: 11px;">Docker</a> <a href="/tags/ElasticSearch/" style="font-size: 17px;">ElasticSearch</a> <a href="/tags/Filter过滤器/" style="font-size: 10px;">Filter过滤器</a> <a href="/tags/Flink/" style="font-size: 18px;">Flink</a> <a href="/tags/GO/" style="font-size: 10px;">GO</a> <a href="/tags/Github-Page/" style="font-size: 10px;">Github Page</a> <a href="/tags/Guava/" style="font-size: 10px;">Guava</a> <a href="/tags/HBase/" style="font-size: 10px;">HBase</a> <a href="/tags/Hibernate-JPA/" style="font-size: 11px;">Hibernate JPA</a> <a href="/tags/IO/" style="font-size: 11px;">IO</a> <a href="/tags/JMM/" style="font-size: 10px;">JMM</a> <a href="/tags/JSON/" style="font-size: 10px;">JSON</a> <a href="/tags/JVM/" style="font-size: 11px;">JVM</a> <a href="/tags/Java/" style="font-size: 20px;">Java</a> <a href="/tags/Kafka/" style="font-size: 12px;">Kafka</a> <a href="/tags/Kibana/" style="font-size: 10px;">Kibana</a> <a href="/tags/LogStash/" style="font-size: 10px;">LogStash</a> <a href="/tags/Mac/" style="font-size: 10px;">Mac</a> <a href="/tags/Maven/" style="font-size: 11px;">Maven</a> <a href="/tags/MySQL/" style="font-size: 12px;">MySQL</a> <a href="/tags/Mybatis/" style="font-size: 14px;">Mybatis</a> <a href="/tags/NIO/" style="font-size: 10px;">NIO</a> <a href="/tags/Netty/" style="font-size: 10px;">Netty</a> <a href="/tags/Nginx/" style="font-size: 11px;">Nginx</a> <a href="/tags/Oracle/" style="font-size: 10px;">Oracle</a> <a href="/tags/Pyspider/" style="font-size: 10px;">Pyspider</a> <a href="/tags/Python/" style="font-size: 12px;">Python</a> <a href="/tags/RabbitMQ/" style="font-size: 11px;">RabbitMQ</a> <a href="/tags/Redis/" style="font-size: 11px;">Redis</a> <a href="/tags/RocketMQ/" style="font-size: 13px;">RocketMQ</a> <a href="/tags/Servlet/" style="font-size: 10px;">Servlet</a> <a href="/tags/Spring/" style="font-size: 12px;">Spring</a> <a href="/tags/Spring-MVC/" style="font-size: 15px;">Spring MVC</a> <a href="/tags/SpringBoot/" style="font-size: 19px;">SpringBoot</a> <a href="/tags/SpringCloud/" style="font-size: 10px;">SpringCloud</a> <a href="/tags/SpringMVC/" style="font-size: 13px;">SpringMVC</a> <a href="/tags/String/" style="font-size: 10px;">String</a> <a href="/tags/Velocity/" style="font-size: 10px;">Velocity</a> <a href="/tags/Zookeeper/" style="font-size: 10px;">Zookeeper</a> <a href="/tags/finally/" style="font-size: 10px;">finally</a> <a href="/tags/foreach/" style="font-size: 10px;">foreach</a> <a href="/tags/hexo/" style="font-size: 12px;">hexo</a> <a href="/tags/lombok/" style="font-size: 10px;">lombok</a> <a href="/tags/lua/" style="font-size: 10px;">lua</a> <a href="/tags/yilia/" style="font-size: 11px;">yilia</a> <a href="/tags/书籍/" style="font-size: 10px;">书籍</a> <a href="/tags/分布式锁/" style="font-size: 10px;">分布式锁</a> <a href="/tags/前端/" style="font-size: 12px;">前端</a> <a href="/tags/励志/" style="font-size: 10px;">励志</a> <a href="/tags/博客合集/" style="font-size: 10px;">博客合集</a> <a href="/tags/博客网站/" style="font-size: 10px;">博客网站</a> <a href="/tags/多线程/" style="font-size: 11px;">多线程</a> <a href="/tags/大数据/" style="font-size: 18px;">大数据</a> <a href="/tags/字符串/" style="font-size: 11px;">字符串</a> <a href="/tags/实习圈/" style="font-size: 10px;">实习圈</a> <a href="/tags/循环队列/" style="font-size: 10px;">循环队列</a> <a href="/tags/微服务/" style="font-size: 11px;">微服务</a> <a href="/tags/性能调优工具/" style="font-size: 10px;">性能调优工具</a> <a href="/tags/投资理财/" style="font-size: 10px;">投资理财</a> <a href="/tags/数据库/" style="font-size: 12px;">数据库</a> <a href="/tags/数据结构/" style="font-size: 13px;">数据结构</a> <a href="/tags/文件/" style="font-size: 10px;">文件</a> <a href="/tags/旋转词/" style="font-size: 10px;">旋转词</a> <a href="/tags/流式计算/" style="font-size: 18px;">流式计算</a> <a href="/tags/流控/" style="font-size: 10px;">流控</a> <a href="/tags/爬虫/" style="font-size: 12px;">爬虫</a> <a href="/tags/算法/" style="font-size: 12px;">算法</a> <a href="/tags/类加载机制/" style="font-size: 10px;">类加载机制</a> <a href="/tags/线程池/" style="font-size: 10px;">线程池</a> <a href="/tags/编码/" style="font-size: 10px;">编码</a> <a href="/tags/表达式/" style="font-size: 10px;">表达式</a> <a href="/tags/邮件发送/" style="font-size: 10px;">邮件发送</a> <a href="/tags/随笔/" style="font-size: 16px;">随笔</a> <a href="/tags/面经/" style="font-size: 16px;">面经</a>
                    </div>
                </section>
                
                
                
                <section class="switch-part switch-part3">
                    <div id="js-friends">
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://blog.csdn.net/tzs_1041218129">CSDN博客</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="https://juejin.im/user/57510b82128fe10056ca70fc">掘金</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.iocoder.cn/?vip">芋道源码</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://fengzhaofeng.net/">冯兆峯</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.codedata.cn/">CodeData</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="https://investguider.com/">美股指南</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.songyawei.cn/">Linux运维工程师</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.carlzone.cn/">carl.zhao</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://dongkelun.com/">伦少的博客</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.jiangxinlingdu.com/">匠心零度</a>
                    
                    </div>
                </section>
                

                
                
                <section class="switch-part switch-part4">
                
                    <div id="js-aboutme">17.08 ~ 18.05 永辉云创基础架构组实习     2018.06 大学毕业   2018.06 ～ 现在  另一家公司各种折腾!</div>
                </section>
                
            </div>
        </div>
        <div>
            <img src="/img/wx.jpg"  alt="zhisheng" />
        </div>
    </header>
</div>
    </div>
    <div class="mid-col">
      <nav id="mobile-nav">
      <div class="overlay">
          <div class="slider-trigger"></div>
          <h1 class="header-author js-mobile-header hide"><a href="/" title="Me">zhisheng</a></h1>
      </div>
    <div class="intrude-less">
        <header id="header" class="inner">
            <a href="/" class="profilepic">
                
                    <img lazy-src="/img/avatar.png" class="js-avatar">
                
            </a>
            <hgroup>
              <h1 class="header-author"><a href="/" title="Me">zhisheng</a></h1>
            </hgroup>
            
            <p class="header-subtitle">坑要一个个填，路要一步步走！</p>
            
            <nav class="header-menu">
                <ul>
                
                    <li><a href="/">主页</a></li>
                
                    <li><a href="/tags/Flink/">Flink</a></li>
                
                    <li><a href="/tags/ElasticSearch/">ElasticSearch</a></li>
                
                    <li><a href="/tags/SpringBoot/">Spring Boot</a></li>
                
                <div class="clearfix"></div>
                </ul>
            </nav>
            <nav class="header-nav">
                <div class="social">
                    
                        <a class="wechat" target="_blank" href="wechat:zhisheng_tian" title="wechat">wechat</a>
                    
                        <a class="mail" target="_blank" href="mailto://zhisheng2018@gmail.com" title="mail">mail</a>
                    
                        <a class="github" target="_blank" href="https://github.com/zhisheng17/" title="github">github</a>
                    
                        <a class="zhihu" target="_blank" href="https://www.zhihu.com/people/tian-zhisheng/activities" title="zhihu">zhihu</a>
                    
                </div>
            </nav>
        </header>                
    </div>
</nav>
      <div class="body-wrap"><article id="post-Elasticsearch-analyzers" class="article article-type-post" itemscope itemprop="blogPost">
  
    <div class="article-meta">
      <a href="/2017/09/08/Elasticsearch-analyzers/" class="article-date">
      <time datetime="2017-09-07T16:00:00.000Z" itemprop="datePublished">2017-09-08</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 class="article-title" itemprop="name">
      Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法
    </h1>
  

      </header>
      
      <div class="article-info article-info-post">
        

        
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/ElasticSearch/">ElasticSearch</a></li></ul>
    </div>

        <div class="clearfix"></div>
      </div>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
        <p>介绍：ElasticSearch 是一个基于 Lucene 的搜索服务器。它提供了一个分布式多用户能力的全文搜索引擎，基于 RESTful web 接口。Elasticsearch 是用 Java 开发的，并作为Apache许可条款下的开放源码发布，是当前流行的企业级搜索引擎。设计用于云计算中，能够达到实时搜索，稳定，可靠，快速，安装使用方便。</p>
<p>Elasticsearch中，内置了很多分词器（analyzers）。下面来进行比较下系统默认分词器和常用的中文分词器之间的区别。<br><a id="more"></a></p>
<h2 id="系统默认分词器："><a href="#系统默认分词器：" class="headerlink" title="系统默认分词器："></a>系统默认分词器：</h2><h3 id="1、standard-分词器"><a href="#1、standard-分词器" class="headerlink" title="1、standard 分词器"></a>1、standard 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-standard-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-standard-analyzer.html</a></p>
<p>如何使用：<a href="http://www.yiibai.com/lucene/lucene_standardanalyzer.html">http://www.yiibai.com/lucene/lucene_standardanalyzer.html</a></p>
<p>英文的处理能力同于StopAnalyzer.支持中文采用的方法为单字切分。他会将词汇单元转换成小写形式，并去除停用词和标点符号。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**StandardAnalyzer分析器*/</span></span><br><span class="line"><span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">standardAnalyzer</span><span class="params">(String msg)</span></span>&#123;</span><br><span class="line">    StandardAnalyzer analyzer = <span class="keyword">new</span> StandardAnalyzer(Version.LUCENE_36);</span><br><span class="line">   <span class="keyword">this</span>.getTokens(analyzer, msg);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="2、simple-分词器"><a href="#2、simple-分词器" class="headerlink" title="2、simple 分词器"></a>2、simple 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-simple-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-simple-analyzer.html</a></p>
<p>如何使用: <a href="http://www.yiibai.com/lucene/lucene_simpleanalyzer.html">http://www.yiibai.com/lucene/lucene_simpleanalyzer.html</a></p>
<p>功能强于WhitespaceAnalyzer, 首先会通过非字母字符来分割文本信息，然后将词汇单元统一为小写形式。该分析器会去掉数字类型的字符。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**SimpleAnalyzer分析器*/</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">simpleAnalyzer</span><span class="params">(String msg)</span></span>&#123;</span><br><span class="line">        SimpleAnalyzer analyzer = <span class="keyword">new</span> SimpleAnalyzer(Version.LUCENE_36);</span><br><span class="line">        <span class="keyword">this</span>.getTokens(analyzer, msg);</span><br><span class="line">    &#125;</span><br></pre></td></tr></table></figure>
<h3 id="3、Whitespace-分词器"><a href="#3、Whitespace-分词器" class="headerlink" title="3、Whitespace 分词器"></a>3、Whitespace 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-whitespace-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-whitespace-analyzer.html</a></p>
<p>如何使用：<a href="http://www.yiibai.com/lucene/lucene_whitespaceanalyzer.html">http://www.yiibai.com/lucene/lucene_whitespaceanalyzer.html</a></p>
<p>仅仅是去除空格，对字符没有lowcase化,不支持中文；<br>并且不对生成的词汇单元进行其他的规范化处理。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**WhitespaceAnalyzer分析器*/</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">whitespaceAnalyzer</span><span class="params">(String msg)</span></span>&#123;</span><br><span class="line">        WhitespaceAnalyzer analyzer = <span class="keyword">new</span> WhitespaceAnalyzer(Version.LUCENE_36);</span><br><span class="line">        <span class="keyword">this</span>.getTokens(analyzer, msg);</span><br><span class="line">    &#125;</span><br></pre></td></tr></table></figure>
<h3 id="4、Stop-分词器"><a href="#4、Stop-分词器" class="headerlink" title="4、Stop 分词器"></a>4、Stop 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stop-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stop-analyzer.html</a></p>
<p>如何使用：<a href="http://www.yiibai.com/lucene/lucene_stopanalyzer.html">http://www.yiibai.com/lucene/lucene_stopanalyzer.html</a></p>
<p> StopAnalyzer的功能超越了SimpleAnalyzer，在SimpleAnalyzer的基础上增加了去除英文中的常用单词（如the，a等），也可以更加自己的需要设置常用单词；不支持中文</p>
 <figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**StopAnalyzer分析器*/</span></span><br><span class="line">   <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">stopAnalyzer</span><span class="params">(String msg)</span></span>&#123;</span><br><span class="line">       StopAnalyzer analyzer = <span class="keyword">new</span> StopAnalyzer(Version.LUCENE_36);</span><br><span class="line">       <span class="keyword">this</span>.getTokens(analyzer, msg);</span><br><span class="line">   &#125;</span><br></pre></td></tr></table></figure>
<h3 id="5、keyword-分词器"><a href="#5、keyword-分词器" class="headerlink" title="5、keyword 分词器"></a>5、keyword 分词器</h3><p>KeywordAnalyzer把整个输入作为一个单独词汇单元，方便特殊类型的文本进行索引和检索。针对邮政编码，地址等文本信息使用关键词分词器进行索引项建立非常方便。</p>
<h3 id="6、pattern-分词器"><a href="#6、pattern-分词器" class="headerlink" title="6、pattern 分词器"></a>6、pattern 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-pattern-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-pattern-analyzer.html</a></p>
<p>一个pattern类型的analyzer可以通过正则表达式将文本分成”terms”(经过token Filter 后得到的东西 )。接受如下设置:</p>
<p>一个 pattern analyzer 可以做如下的属性设置:</p>
<table>
<thead>
<tr>
<th>lowercase</th>
<th>terms是否是小写. 默认为 true 小写.</th>
</tr>
</thead>
<tbody>
<tr>
<td>pattern</td>
<td>正则表达式的pattern, 默认是 \W+.</td>
</tr>
<tr>
<td>flags</td>
<td>正则表达式的flags</td>
</tr>
<tr>
<td>stopwords</td>
<td>一个用于初始化stop filter的需要stop 单词的列表.默认单词是空的列表</td>
</tr>
</tbody>
</table>
<h3 id="7、language-分词器"><a href="#7、language-分词器" class="headerlink" title="7、language 分词器"></a>7、language 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-lang-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-lang-analyzer.html</a></p>
<p>一个用于解析特殊语言文本的analyzer集合。（ arabic,armenian, basque, brazilian, bulgarian, catalan, cjk, czech, danish, dutch, english, finnish, french,galician, german, greek, hindi, hungarian, indonesian, irish, italian, latvian, lithuanian, norwegian,persian, portuguese, romanian, russian, sorani, spanish, swedish, turkish, thai.）可惜没有中文。不予考虑</p>
<h3 id="8、snowball-分词器"><a href="#8、snowball-分词器" class="headerlink" title="8、snowball 分词器"></a>8、snowball 分词器</h3><p>一个snowball类型的analyzer是由standard tokenizer和standard filter、lowercase filter、stop filter、snowball filter这四个filter构成的。</p>
<p>snowball analyzer 在Lucene中通常是不推荐使用的。</p>
<h3 id="9、Custom-分词器"><a href="#9、Custom-分词器" class="headerlink" title="9、Custom 分词器"></a>9、Custom 分词器</h3><p>是自定义的analyzer。允许多个零到多个tokenizer，零到多个 Char Filters. custom analyzer 的名字不能以 “_”开头.</p>
<p>The following are settings that can be set for a custom analyzer type:</p>
<table>
<thead>
<tr>
<th>Setting</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>tokenizer</td>
<td>通用的或者注册的tokenizer.</td>
</tr>
<tr>
<td>filter</td>
<td>通用的或者注册的token filters</td>
</tr>
<tr>
<td>char_filter</td>
<td>通用的或者注册的 character filters</td>
</tr>
<tr>
<td>position_increment_gap</td>
<td>距离查询时，最大允许查询的距离，默认是100</td>
</tr>
</tbody>
</table>
<p>自定义的模板：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line">index :</span><br><span class="line">    analysis :</span><br><span class="line">        analyzer :</span><br><span class="line">            myAnalyzer2 :</span><br><span class="line">                type : custom</span><br><span class="line">                tokenizer : myTokenizer1</span><br><span class="line">                filter : [myTokenFilter1, myTokenFilter2]</span><br><span class="line">                char_filter : [my_html]</span><br><span class="line">                position_increment_gap: 256</span><br><span class="line">        tokenizer :</span><br><span class="line">            myTokenizer1 :</span><br><span class="line">                type : standard</span><br><span class="line">                max_token_length : 900</span><br><span class="line">        filter :</span><br><span class="line">            myTokenFilter1 :</span><br><span class="line">                type : stop</span><br><span class="line">                stopwords : [stop1, stop2, stop3, stop4]</span><br><span class="line">            myTokenFilter2 :</span><br><span class="line">                type : length</span><br><span class="line">                min : 0</span><br><span class="line">                max : 2000</span><br><span class="line">        char_filter :</span><br><span class="line">              my_html :</span><br><span class="line">                type : html_strip</span><br><span class="line">                escaped_tags : [xxx, yyy]</span><br><span class="line">                read_ahead : 1024</span><br></pre></td></tr></table></figure>
<h3 id="10、fingerprint-分词器"><a href="#10、fingerprint-分词器" class="headerlink" title="10、fingerprint 分词器"></a>10、fingerprint 分词器</h3><p><a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-fingerprint-analyzer.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-fingerprint-analyzer.html</a></p>
<hr>
<h2 id="中文分词器："><a href="#中文分词器：" class="headerlink" title="中文分词器："></a>中文分词器：</h2><h3 id="1、ik-analyzer"><a href="#1、ik-analyzer" class="headerlink" title="1、ik-analyzer"></a>1、ik-analyzer</h3><p><a href="https://github.com/wks/ik-analyzer">https://github.com/wks/ik-analyzer</a></p>
<p>IKAnalyzer是一个开源的，基于java语言开发的轻量级的中文分词工具包。</p>
<p>采用了特有的“正向迭代最细粒度切分算法“，支持细粒度和最大词长两种切分模式；具有83万字/秒（1600KB/S）的高速处理能力。</p>
<p>采用了多子处理器分析模式，支持：英文字母、数字、中文词汇等分词处理，兼容韩文、日文字符</p>
<p>优化的词典存储，更小的内存占用。支持用户词典扩展定义</p>
<p>针对Lucene全文检索优化的查询分析器IKQueryParser(作者吐血推荐)；引入简单搜索表达式，采用歧义分析算法优化查询关键字的搜索排列组合，能极大的提高Lucene检索的命中率。</p>
<p>Maven用法：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.wltea.ik-analyzer<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>ik-analyzer<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">version</span>&gt;</span>3.2.8<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br></pre></td></tr></table></figure>
<p>在IK Analyzer加入Maven Central Repository之前，你需要手动安装，安装到本地的repository，或者上传到自己的Maven repository服务器上。</p>
<p>要安装到本地Maven repository，使用如下命令，将自动编译，打包并安装：<br>mvn install -Dmaven.test.skip=true</p>
<h4 id="Elasticsearch添加中文分词"><a href="#Elasticsearch添加中文分词" class="headerlink" title="Elasticsearch添加中文分词"></a>Elasticsearch添加中文分词</h4><h5 id="安装IK分词插件"><a href="#安装IK分词插件" class="headerlink" title="安装IK分词插件"></a>安装IK分词插件</h5><p><a href="https://github.com/medcl/elasticsearch-analysis-ik">https://github.com/medcl/elasticsearch-analysis-ik</a></p>
<p>进入elasticsearch-analysis-ik-master</p>
<p>更多安装请参考博客：</p>
<p>1、<a href="http://blog.csdn.net/dingzfang/article/details/42776693">为elastic添加中文分词</a> ： <a href="http://blog.csdn.net/dingzfang/article/details/42776693">http://blog.csdn.net/dingzfang/article/details/42776693</a></p>
<p>2、<a href="http://www.cnblogs.com/xing901022/p/5910139.html">如何在Elasticsearch中安装中文分词器(IK+pinyin)</a> ：<a href="http://www.cnblogs.com/xing901022/p/5910139.html">http://www.cnblogs.com/xing901022/p/5910139.html</a></p>
<p>3、<a href="http://blog.csdn.net/jam00/article/details/52983056">Elasticsearch 中文分词器 IK 配置和使用</a> ： <a href="http://blog.csdn.net/jam00/article/details/52983056">http://blog.csdn.net/jam00/article/details/52983056</a></p>
<h4 id="ik-带有两个分词器"><a href="#ik-带有两个分词器" class="headerlink" title="ik 带有两个分词器"></a>ik 带有两个分词器</h4><p><strong>ik_max_word</strong> ：会将文本做最细粒度的拆分；尽可能多的拆分出词语</p>
<p><strong>ik_smart</strong>：会做最粗粒度的拆分；已被分出的词语将不会再次被其它词语占有</p>
<p>区别：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br></pre></td><td class="code"><pre><span class="line"># ik_max_word</span><br><span class="line"></span><br><span class="line">curl -XGET &apos;http://localhost:9200/_analyze?pretty&amp;analyzer=ik_max_word&apos; -d &apos;联想是全球最大的笔记本厂商&apos;</span><br><span class="line">#返回</span><br><span class="line"></span><br><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;联想&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 0,</span><br><span class="line">      &quot;end_offset&quot; : 2,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 0</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;是&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 2,</span><br><span class="line">      &quot;end_offset&quot; : 3,</span><br><span class="line">      &quot;type&quot; : &quot;CN_CHAR&quot;,</span><br><span class="line">      &quot;position&quot; : 1</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;全球&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 3,</span><br><span class="line">      &quot;end_offset&quot; : 5,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 2</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;最大&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 5,</span><br><span class="line">      &quot;end_offset&quot; : 7,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 3</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;的&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 7,</span><br><span class="line">      &quot;end_offset&quot; : 8,</span><br><span class="line">      &quot;type&quot; : &quot;CN_CHAR&quot;,</span><br><span class="line">      &quot;position&quot; : 4</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;笔记本&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 8,</span><br><span class="line">      &quot;end_offset&quot; : 11,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 5</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;笔记&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 8,</span><br><span class="line">      &quot;end_offset&quot; : 10,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 6</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;本厂&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 10,</span><br><span class="line">      &quot;end_offset&quot; : 12,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 7</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;厂商&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 11,</span><br><span class="line">      &quot;end_offset&quot; : 13,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 8</span><br><span class="line">    &#125;</span><br><span class="line">  ]</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># ik_smart</span><br><span class="line"></span><br><span class="line">curl -XGET &apos;http://localhost:9200/_analyze?pretty&amp;analyzer=ik_smart&apos; -d &apos;联想是全球最大的笔记本厂商&apos;</span><br><span class="line"></span><br><span class="line"># 返回</span><br><span class="line"></span><br><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;联想&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 0,</span><br><span class="line">      &quot;end_offset&quot; : 2,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 0</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;是&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 2,</span><br><span class="line">      &quot;end_offset&quot; : 3,</span><br><span class="line">      &quot;type&quot; : &quot;CN_CHAR&quot;,</span><br><span class="line">      &quot;position&quot; : 1</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;全球&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 3,</span><br><span class="line">      &quot;end_offset&quot; : 5,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 2</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;最大&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 5,</span><br><span class="line">      &quot;end_offset&quot; : 7,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 3</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;的&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 7,</span><br><span class="line">      &quot;end_offset&quot; : 8,</span><br><span class="line">      &quot;type&quot; : &quot;CN_CHAR&quot;,</span><br><span class="line">      &quot;position&quot; : 4</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;笔记本&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 8,</span><br><span class="line">      &quot;end_offset&quot; : 11,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 5</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;厂商&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 11,</span><br><span class="line">      &quot;end_offset&quot; : 13,</span><br><span class="line">      &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">      &quot;position&quot; : 6</span><br><span class="line">    &#125;</span><br><span class="line">  ]</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>下面我们来创建一个索引，使用 ik<br>创建一个名叫 iktest 的索引，设置它的分析器用 ik ，分词器用 ik_max_word，并创建一个 article 的类型，里面有一个 subject 的字段，指定其使用 ik_max_word 分词器</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">curl -XPUT &apos;http://localhost:9200/iktest?pretty&apos; -d &apos;&#123;</span><br><span class="line">    &quot;settings&quot; : &#123;</span><br><span class="line">        &quot;analysis&quot; : &#123;</span><br><span class="line">            &quot;analyzer&quot; : &#123;</span><br><span class="line">                &quot;ik&quot; : &#123;</span><br><span class="line">                    &quot;tokenizer&quot; : &quot;ik_max_word&quot;</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;,</span><br><span class="line">    &quot;mappings&quot; : &#123;</span><br><span class="line">        &quot;article&quot; : &#123;</span><br><span class="line">            &quot;dynamic&quot; : true,</span><br><span class="line">            &quot;properties&quot; : &#123;</span><br><span class="line">                &quot;subject&quot; : &#123;</span><br><span class="line">                    &quot;type&quot; : &quot;string&quot;,</span><br><span class="line">                    &quot;analyzer&quot; : &quot;ik_max_word&quot;</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;&apos;</span><br></pre></td></tr></table></figure>
<p>批量添加几条数据，这里我指定元数据 _id 方便查看，subject 内容为我随便找的几条新闻的标题</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/iktest/article/_bulk?pretty -d &apos;</span><br><span class="line">&#123; &quot;index&quot; : &#123; &quot;_id&quot; : &quot;1&quot; &#125; &#125;</span><br><span class="line">&#123;&quot;subject&quot; : &quot;＂闺蜜＂崔顺实被韩检方传唤 韩总统府促彻查真相&quot; &#125;</span><br><span class="line">&#123; &quot;index&quot; : &#123; &quot;_id&quot; : &quot;2&quot; &#125; &#125;</span><br><span class="line">&#123;&quot;subject&quot; : &quot;韩举行＂护国训练＂ 青瓦台:决不许国家安全出问题&quot; &#125;</span><br><span class="line">&#123; &quot;index&quot; : &#123; &quot;_id&quot; : &quot;3&quot; &#125; &#125;</span><br><span class="line">&#123;&quot;subject&quot; : &quot;媒体称FBI已经取得搜查令 检视希拉里电邮&quot; &#125;</span><br><span class="line">&#123; &quot;index&quot; : &#123; &quot;_id&quot; : &quot;4&quot; &#125; &#125;</span><br><span class="line">&#123;&quot;subject&quot; : &quot;村上春树获安徒生奖 演讲中谈及欧洲排外问题&quot; &#125;</span><br><span class="line">&#123; &quot;index&quot; : &#123; &quot;_id&quot; : &quot;5&quot; &#125; &#125;</span><br><span class="line">&#123;&quot;subject&quot; : &quot;希拉里团队炮轰FBI 参院民主党领袖批其“违法”&quot; &#125;</span><br><span class="line">&apos;</span><br></pre></td></tr></table></figure>
<p>查询 “希拉里和韩国”</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/iktest/article/_search?pretty  -d&apos;</span><br><span class="line">&#123;</span><br><span class="line">    &quot;query&quot; : &#123; &quot;match&quot; : &#123; &quot;subject&quot; : &quot;希拉里和韩国&quot; &#125;&#125;,</span><br><span class="line">    &quot;highlight&quot; : &#123;</span><br><span class="line">        &quot;pre_tags&quot; : [&quot;&lt;font color=&apos;red&apos;&gt;&quot;],</span><br><span class="line">        &quot;post_tags&quot; : [&quot;&lt;/font&gt;&quot;],</span><br><span class="line">        &quot;fields&quot; : &#123;</span><br><span class="line">            &quot;subject&quot; : &#123;&#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line">&apos;</span><br><span class="line">#返回</span><br><span class="line">&#123;</span><br><span class="line">  &quot;took&quot; : 113,</span><br><span class="line">  &quot;timed_out&quot; : false,</span><br><span class="line">  &quot;_shards&quot; : &#123;</span><br><span class="line">    &quot;total&quot; : 5,</span><br><span class="line">    &quot;successful&quot; : 5,</span><br><span class="line">    &quot;failed&quot; : 0</span><br><span class="line">  &#125;,</span><br><span class="line">  &quot;hits&quot; : &#123;</span><br><span class="line">    &quot;total&quot; : 4,</span><br><span class="line">    &quot;max_score&quot; : 0.034062363,</span><br><span class="line">    &quot;hits&quot; : [ &#123;</span><br><span class="line">      &quot;_index&quot; : &quot;iktest&quot;,</span><br><span class="line">      &quot;_type&quot; : &quot;article&quot;,</span><br><span class="line">      &quot;_id&quot; : &quot;2&quot;,</span><br><span class="line">      &quot;_score&quot; : 0.034062363,</span><br><span class="line">      &quot;_source&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : &quot;韩举行＂护国训练＂ 青瓦台:决不许国家安全出问题&quot;</span><br><span class="line">      &#125;,</span><br><span class="line">      &quot;highlight&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : [ &quot;&lt;font color=red&gt;韩&lt;/font&gt;举行＂护&lt;font color=red&gt;国&lt;/font&gt;训练＂ 青瓦台:决不许国家安全出问题&quot; ]</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;, &#123;</span><br><span class="line">      &quot;_index&quot; : &quot;iktest&quot;,</span><br><span class="line">      &quot;_type&quot; : &quot;article&quot;,</span><br><span class="line">      &quot;_id&quot; : &quot;3&quot;,</span><br><span class="line">      &quot;_score&quot; : 0.0076681254,</span><br><span class="line">      &quot;_source&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : &quot;媒体称FBI已经取得搜查令 检视希拉里电邮&quot;</span><br><span class="line">      &#125;,</span><br><span class="line">      &quot;highlight&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : [ &quot;媒体称FBI已经取得搜查令 检视&lt;font color=red&gt;希拉里&lt;/font&gt;电邮&quot; ]</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;, &#123;</span><br><span class="line">      &quot;_index&quot; : &quot;iktest&quot;,</span><br><span class="line">      &quot;_type&quot; : &quot;article&quot;,</span><br><span class="line">      &quot;_id&quot; : &quot;5&quot;,</span><br><span class="line">      &quot;_score&quot; : 0.006709609,</span><br><span class="line">      &quot;_source&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : &quot;希拉里团队炮轰FBI 参院民主党领袖批其“违法”&quot;</span><br><span class="line">      &#125;,</span><br><span class="line">      &quot;highlight&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : [ &quot;&lt;font color=red&gt;希拉里&lt;/font&gt;团队炮轰FBI 参院民主党领袖批其“违法”&quot; ]</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;, &#123;</span><br><span class="line">      &quot;_index&quot; : &quot;iktest&quot;,</span><br><span class="line">      &quot;_type&quot; : &quot;article&quot;,</span><br><span class="line">      &quot;_id&quot; : &quot;1&quot;,</span><br><span class="line">      &quot;_score&quot; : 0.0021509775,</span><br><span class="line">      &quot;_source&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : &quot;＂闺蜜＂崔顺实被韩检方传唤 韩总统府促彻查真相&quot;</span><br><span class="line">      &#125;,</span><br><span class="line">      &quot;highlight&quot; : &#123;</span><br><span class="line">        &quot;subject&quot; : [ &quot;＂闺蜜＂崔顺实被&lt;font color=red&gt;韩&lt;/font&gt;检方传唤 &lt;font color=red&gt;韩&lt;/font&gt;总统府促彻查真相&quot; ]</span><br><span class="line">      &#125;</span><br><span class="line">    &#125; ]</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>这里用了高亮属性 highlight，直接显示到 html 中，被匹配到的字或词将以红色突出显示。若要用过滤搜索，直接将 match 改为 term 即可</p>
<h4 id="热词更新配置"><a href="#热词更新配置" class="headerlink" title="热词更新配置"></a>热词更新配置</h4><p>网络词语日新月异，如何让新出的网络热词（或特定的词语）实时的更新到我们的搜索当中呢</p>
<p>先用 ik 测试一下</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line">curl -XGET &apos;http://localhost:9200/_analyze?pretty&amp;analyzer=ik_max_word&apos; -d &apos;</span><br><span class="line">成龙原名陈港生</span><br><span class="line">&apos;</span><br><span class="line">#返回</span><br><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [ &#123;</span><br><span class="line">    &quot;token&quot; : &quot;成龙&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 1,</span><br><span class="line">    &quot;end_offset&quot; : 3,</span><br><span class="line">    &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">    &quot;position&quot; : 0</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;原名&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 3,</span><br><span class="line">    &quot;end_offset&quot; : 5,</span><br><span class="line">    &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">    &quot;position&quot; : 1</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;陈&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 5,</span><br><span class="line">    &quot;end_offset&quot; : 6,</span><br><span class="line">    &quot;type&quot; : &quot;CN_CHAR&quot;,</span><br><span class="line">    &quot;position&quot; : 2</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;港&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 6,</span><br><span class="line">    &quot;end_offset&quot; : 7,</span><br><span class="line">    &quot;type&quot; : &quot;CN_WORD&quot;,</span><br><span class="line">    &quot;position&quot; : 3</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;生&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 7,</span><br><span class="line">    &quot;end_offset&quot; : 8,</span><br><span class="line">    &quot;type&quot; : &quot;CN_CHAR&quot;,</span><br><span class="line">    &quot;position&quot; : 4</span><br><span class="line">  &#125; ]</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>ik 的主词典中没有”陈港生” 这个词，所以被拆分了。<br>现在我们来配置一下</p>
<p>修改 IK 的配置文件 ：ES 目录/plugins/ik/config/ik/IKAnalyzer.cfg.xml</p>
<p>修改如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span><br><span class="line">&lt;!DOCTYPE properties SYSTEM &quot;http://java.sun.com/dtd/properties.dtd&quot;&gt;</span><br><span class="line">&lt;properties&gt;</span><br><span class="line">    &lt;comment&gt;IK Analyzer 扩展配置&lt;/comment&gt;</span><br><span class="line">    &lt;!--用户可以在这里配置自己的扩展字典 --&gt;</span><br><span class="line">    &lt;entry key=&quot;ext_dict&quot;&gt;custom/mydict.dic;custom/single_word_low_freq.dic&lt;/entry&gt;</span><br><span class="line">     &lt;!--用户可以在这里配置自己的扩展停止词字典--&gt;</span><br><span class="line">    &lt;entry key=&quot;ext_stopwords&quot;&gt;custom/ext_stopword.dic&lt;/entry&gt;</span><br><span class="line">    &lt;!--用户可以在这里配置远程扩展字典 --&gt;</span><br><span class="line">    &lt;entry key=&quot;remote_ext_dict&quot;&gt;http://192.168.1.136/hotWords.php&lt;/entry&gt;</span><br><span class="line">    &lt;!--用户可以在这里配置远程扩展停止词字典--&gt;</span><br><span class="line">    &lt;!-- &lt;entry key=&quot;remote_ext_stopwords&quot;&gt;words_location&lt;/entry&gt; --&gt;</span><br><span class="line">&lt;/properties&gt;</span><br></pre></td></tr></table></figure>
<p>这里我是用的是远程扩展字典，因为可以使用其他程序调用更新，且不用重启 ES，很方便；当然使用自定义的 mydict.dic 字典也是很方便的，一行一个词，自己加就可以了</p>
<p>既然是远程词典，那么就要是一个可访问的链接，可以是一个页面，也可以是一个txt的文档，但要保证输出的内容是 utf-8 的格式</p>
<p>hotWords.php 的内容</p>
<figure class="highlight php"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$s = <span class="string">&lt;&lt;&lt;'EOF'</span></span><br><span class="line"><span class="string">陈港生</span></span><br><span class="line"><span class="string">元楼</span></span><br><span class="line"><span class="string">蓝瘦</span></span><br><span class="line"><span class="string">EOF;</span></span><br><span class="line">header(<span class="string">'Last-Modified: '</span>.gmdate(<span class="string">'D, d M Y H:i:s'</span>, time()).<span class="string">' GMT'</span>, <span class="keyword">true</span>, <span class="number">200</span>);</span><br><span class="line">header(<span class="string">'ETag: "5816f349-19"'</span>);</span><br><span class="line"><span class="keyword">echo</span> $s;</span><br></pre></td></tr></table></figure>
<p>ik 接收两个返回的头部属性 Last-Modified 和 ETag，只要其中一个有变化，就会触发更新，ik 会每分钟获取一次<br>重启 Elasticsearch ，查看启动记录，看到了三个词已被加载进来</p>
<p>再次执行上面的请求，返回, 就可以看到 ik 分词器已经匹配到了 “陈港生” 这个词，同理一些关于我们公司的专有名字（例如：永辉、永辉超市、永辉云创、云创 …. ）也可以自己手动添加到字典中去。</p>
<h3 id="2、结巴中文分词"><a href="#2、结巴中文分词" class="headerlink" title="2、结巴中文分词"></a>2、结巴中文分词</h3><h4 id="特点："><a href="#特点：" class="headerlink" title="特点："></a>特点：</h4><p>1、支持三种分词模式：</p>
<ul>
<li><p>精确模式，试图将句子最精确地切开，适合文本分析；</p>
</li>
<li><p>全模式，把句子中所有的可以成词的词语都扫描出来, 速度非常快，但是不能解决歧义；</p>
</li>
<li><p>搜索引擎模式，在精确模式的基础上，对长词再次切分，提高召回率，适合用于搜索引擎分词。</p>
</li>
</ul>
<p>2、支持繁体分词</p>
<p>3、支持自定义词典</p>
<h3 id="3、THULAC"><a href="#3、THULAC" class="headerlink" title="3、THULAC"></a>3、THULAC</h3><p>THULAC（THU Lexical Analyzer for Chinese）由清华大学自然语言处理与社会人文计算实验室研制推出的一套中文词法分析工具包，具有中文分词和词性标注功能。THULAC具有如下几个特点：</p>
<p>能力强。利用我们集成的目前世界上规模最大的人工分词和词性标注中文语料库（约含5800万字）训练而成，模型标注能力强大。</p>
<p>准确率高。该工具包在标准数据集Chinese Treebank（CTB5）上分词的F1值可达97.3％，词性标注的F1值可达到92.9％，与该数据集上最好方法效果相当。</p>
<p>速度较快。同时进行分词和词性标注速度为300KB/s，每秒可处理约15万字。只进行分词速度可达到1.3MB/s。</p>
<p>中文分词工具thulac4j发布</p>
<p>1、规范化分词词典，并去掉一些无用词；</p>
<p>2、重写DAT（双数组Trie树）的构造算法，生成的DAT size减少了8%左右，从而节省了内存；</p>
<p>3、优化分词算法，提高了分词速率。</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>io.github.yizhiru<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>thulac4j<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">version</span>&gt;</span>$&#123;thulac4j.version&#125;<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br></pre></td></tr></table></figure>
<p><a href="http://www.cnblogs.com/en-heng/p/6526598.html">http://www.cnblogs.com/en-heng/p/6526598.html</a></p>
<p>thulac4j支持两种分词模式：</p>
<p>SegOnly模式，只分词没有词性标注；</p>
<p>SegPos模式，分词兼有词性标注。</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// SegOnly mode</span></span><br><span class="line">String sentence = <span class="string">"滔滔的流水，向着波士顿湾无声逝去"</span>;</span><br><span class="line">SegOnly seg = <span class="keyword">new</span> SegOnly(<span class="string">"models/seg_only.bin"</span>);</span><br><span class="line">System.out.println(seg.segment(sentence));</span><br><span class="line"><span class="comment">// [滔滔, 的, 流水, ，, 向着, 波士顿湾, 无声, 逝去]</span></span><br><span class="line"></span><br><span class="line"><span class="comment">// SegPos mode</span></span><br><span class="line">SegPos pos = <span class="keyword">new</span> SegPos(<span class="string">"models/seg_pos.bin"</span>);</span><br><span class="line">System.out.println(pos.segment(sentence));</span><br><span class="line"><span class="comment">//[滔滔/a, 的/u, 流水/n, ，/w, 向着/p, 波士顿湾/ns, 无声/v, 逝去/v]</span></span><br></pre></td></tr></table></figure>
<h3 id="4、NLPIR"><a href="#4、NLPIR" class="headerlink" title="4、NLPIR"></a>4、NLPIR</h3><p>中科院计算所 NLPIR：<a href="http://ictclas.nlpir.org/nlpir/">http://ictclas.nlpir.org/nlpir/</a>  (可直接在线分析中文)</p>
<p>下载地址：<a href="https://github.com/NLPIR-team/NLPIR">https://github.com/NLPIR-team/NLPIR</a></p>
<p>中科院分词系统(NLPIR)JAVA简易教程: <a href="http://www.cnblogs.com/wukongjiuwo/p/4092480.html">http://www.cnblogs.com/wukongjiuwo/p/4092480.html</a></p>
<h3 id="5、ansj分词器"><a href="#5、ansj分词器" class="headerlink" title="5、ansj分词器"></a>5、ansj分词器</h3><p><a href="https://github.com/NLPchina/ansj_seg">https://github.com/NLPchina/ansj_seg</a></p>
<p>这是一个基于n-Gram+CRF+HMM的中文分词的java实现.</p>
<p>分词速度达到每秒钟大约200万字左右（mac air下测试），准确率能达到96%以上</p>
<p>目前实现了.中文分词. 中文姓名识别 .</p>
<p>用户自定义词典,关键字提取，自动摘要，关键字标记等功能<br>可以应用到自然语言处理等方面,适用于对分词效果要求高的各种项目.</p>
<p>maven 引入：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.ansj<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>ansj_seg<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">version</span>&gt;</span>5.1.1<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br></pre></td></tr></table></figure>
<p><strong>调用demo</strong></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">String str = <span class="string">"欢迎使用ansj_seg,(ansj中文分词)在这里如果你遇到什么问题都可以联系我.我一定尽我所能.帮助大家.ansj_seg更快,更准,更自由!"</span> ;</span><br><span class="line"> System.out.println(ToAnalysis.parse(str));</span><br><span class="line"></span><br><span class="line"> 欢迎/v,使用/v,ansj/en,_,seg/en,,,(,ansj/en,中文/nz,分词/n,),在/p,这里/r,如果/c,你/r,遇到/v,什么/r,问题/n,都/d,可以/v,联系/v,我/r,./m,我/r,一定/d,尽我所能/l,./m,帮助/v,大家/r,./m,ansj/en,_,seg/en,更快/d,,,更/d,准/a,,,更/d,自由/a,!</span><br></pre></td></tr></table></figure>
<h3 id="6、哈工大的LTP"><a href="#6、哈工大的LTP" class="headerlink" title="6、哈工大的LTP"></a>6、哈工大的LTP</h3><p><a href="https://link.zhihu.com/?target=https%3A//github.com/HIT-SCIR/ltp">https://link.zhihu.com/?target=https%3A//github.com/HIT-SCIR/ltp</a></p>
<p>LTP制定了基于XML的语言处理结果表示，并在此基础上提供了一整套自底向上的丰富而且高效的中文语言处理模块（包括词法、句法、语义等6项中文处理核心技术），以及基于动态链接库（Dynamic Link Library, DLL）的应用程序接口、可视化工具，并且能够以网络服务（Web Service）的形式进行使用。</p>
<p>关于LTP的使用，请参考:  <a href="http://ltp.readthedocs.io/zh_CN/latest/">http://ltp.readthedocs.io/zh_CN/latest/</a></p>
<h3 id="7、庖丁解牛"><a href="#7、庖丁解牛" class="headerlink" title="7、庖丁解牛"></a>7、庖丁解牛</h3><p>下载地址：<a href="http://pan.baidu.com/s/1eQ88SZS">http://pan.baidu.com/s/1eQ88SZS</a></p>
<p>使用分为如下几步：</p>
<ol>
<li><p>配置dic文件：<br>修改paoding-analysis.jar中的paoding-dic-home.properties文件，将“#paoding.dic.home=dic”的注释去掉，并配置成自己dic文件的本地存放路径。eg：/home/hadoop/work/paoding-analysis-2.0.4-beta/dic</p>
</li>
<li><p>把Jar包导入到项目中：<br>将paoding-analysis.jar、commons-logging.jar、lucene-analyzers-2.2.0.jar和lucene-core-2.2.0.jar四个包导入到项目中，这时就可以在代码片段中使用庖丁解牛工具提供的中文分词技术，例如：</p>
</li>
</ol>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">Analyzer analyzer = <span class="keyword">new</span> PaodingAnalyzer(); <span class="comment">//定义一个解析器</span></span><br><span class="line">String text = <span class="string">"庖丁系统是个完全基于lucene的中文分词系统，它就是重新建了一个analyzer，叫做PaodingAnalyzer，这个analyer的核心任务就是生成一个可以切词TokenStream。"</span>; &lt;span style=<span class="string">"font-family: Arial, Helvetica, sans-serif;"</span>&gt;<span class="comment">//待分词的内容&lt;/span&gt;</span></span><br><span class="line">TokenStream tokenStream = analyzer.tokenStream(text, <span class="keyword">new</span> StringReader(text)); <span class="comment">//得到token序列的输出流</span></span><br><span class="line"><span class="keyword">try</span> &#123;</span><br><span class="line">    Token t;</span><br><span class="line">    <span class="keyword">while</span> ((t = tokenStream.next()) != <span class="keyword">null</span>)</span><br><span class="line">    &#123;</span><br><span class="line">           System.out.println(t); <span class="comment">//输出每个token</span></span><br><span class="line">    &#125;</span><br><span class="line">&#125; <span class="keyword">catch</span> (IOException e) &#123;</span><br><span class="line">    e.printStackTrace();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="8、sogo在线分词"><a href="#8、sogo在线分词" class="headerlink" title="8、sogo在线分词"></a>8、sogo在线分词</h3><p>sogo在线分词采用了基于汉字标注的分词方法，主要使用了线性链链CRF（Linear-chain CRF）模型。词性标注模块主要基于结构化线性模型（Structured Linear Model）</p>
<p>在线使用地址为：<br><a href="http://www.sogou.com/labs/webservice/">http://www.sogou.com/labs/webservice/</a></p>
<h3 id="9、word分词"><a href="#9、word分词" class="headerlink" title="9、word分词"></a>9、word分词</h3><p>地址： <a href="https://github.com/ysc/word">https://github.com/ysc/word</a></p>
<p>word分词是一个Java实现的分布式的中文分词组件，提供了多种基于词典的分词算法，并利用ngram模型来消除歧义。能准确识别英文、数字，以及日期、时间等数量词，能识别人名、地名、组织机构名等未登录词。能通过自定义配置文件来改变组件行为，能自定义用户词库、自动检测词库变化、支持大规模分布式环境，能灵活指定多种分词算法，能使用refine功能灵活控制分词结果，还能使用词频统计、词性标注、同义标注、反义标注、拼音标注等功能。提供了10种分词算法，还提供了10种文本相似度算法，同时还无缝和Lucene、Solr、ElasticSearch、Luke集成。注意：word1.3需要JDK1.8</p>
<p>maven 中引入依赖：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">dependencies</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>org.apdplat<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>word<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">version</span>&gt;</span>1.3<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">dependencies</span>&gt;</span></span><br></pre></td></tr></table></figure>
<p>ElasticSearch插件：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br></pre></td><td class="code"><pre><span class="line">1、打开命令行并切换到elasticsearch的bin目录</span><br><span class="line">cd elasticsearch-2.1.1/bin</span><br><span class="line"></span><br><span class="line">2、运行plugin脚本安装word分词插件：</span><br><span class="line">./plugin install http://apdplat.org/word/archive/v1.4.zip</span><br><span class="line"></span><br><span class="line">安装的时候注意：</span><br><span class="line">    如果提示：</span><br><span class="line">        ERROR: failed to download</span><br><span class="line">    或者</span><br><span class="line">        Failed to install word, reason: failed to download</span><br><span class="line">    或者</span><br><span class="line">        ERROR: incorrect hash (SHA1)</span><br><span class="line">    则重新再次运行命令，如果还是不行，多试两次</span><br><span class="line"></span><br><span class="line">如果是elasticsearch1.x系列版本，则使用如下命令：</span><br><span class="line">./plugin -u http://apdplat.org/word/archive/v1.3.1.zip -i word</span><br><span class="line"></span><br><span class="line">3、修改文件elasticsearch-2.1.1/config/elasticsearch.yml，新增如下配置：</span><br><span class="line">index.analysis.analyzer.default.type : &quot;word&quot;</span><br><span class="line">index.analysis.tokenizer.default.type : &quot;word&quot;</span><br><span class="line"></span><br><span class="line">4、启动ElasticSearch测试效果，在Chrome浏览器中访问：</span><br><span class="line">http://localhost:9200/_analyze?analyzer=word&amp;text=杨尚川是APDPlat应用级产品开发平台的作者</span><br><span class="line"></span><br><span class="line">5、自定义配置</span><br><span class="line">修改配置文件elasticsearch-2.1.1/plugins/word/word.local.conf</span><br><span class="line"></span><br><span class="line">6、指定分词算法</span><br><span class="line">修改文件elasticsearch-2.1.1/config/elasticsearch.yml，新增如下配置：</span><br><span class="line">index.analysis.analyzer.default.segAlgorithm : &quot;ReverseMinimumMatching&quot;</span><br><span class="line">index.analysis.tokenizer.default.segAlgorithm : &quot;ReverseMinimumMatching&quot;</span><br><span class="line"></span><br><span class="line">这里segAlgorithm可指定的值有：</span><br><span class="line">正向最大匹配算法：MaximumMatching</span><br><span class="line">逆向最大匹配算法：ReverseMaximumMatching</span><br><span class="line">正向最小匹配算法：MinimumMatching</span><br><span class="line">逆向最小匹配算法：ReverseMinimumMatching</span><br><span class="line">双向最大匹配算法：BidirectionalMaximumMatching</span><br><span class="line">双向最小匹配算法：BidirectionalMinimumMatching</span><br><span class="line">双向最大最小匹配算法：BidirectionalMaximumMinimumMatching</span><br><span class="line">全切分算法：FullSegmentation</span><br><span class="line">最少词数算法：MinimalWordCount</span><br><span class="line">最大Ngram分值算法：MaxNgramScore</span><br><span class="line">如不指定，默认使用双向最大匹配算法：BidirectionalMaximumMatching</span><br></pre></td></tr></table></figure>
<h3 id="10、jcseg分词器"><a href="#10、jcseg分词器" class="headerlink" title="10、jcseg分词器"></a>10、jcseg分词器</h3><p><a href="https://code.google.com/archive/p/jcseg/">https://code.google.com/archive/p/jcseg/</a></p>
<h3 id="11、stanford分词器"><a href="#11、stanford分词器" class="headerlink" title="11、stanford分词器"></a>11、stanford分词器</h3><p>Stanford大学的一个开源分词工具，目前已支持汉语。</p>
<p>首先，去【1】下载Download Stanford Word Segmenter version 3.5.2，取得里面的 data 文件夹，放在maven project的 src/main/resources 里。</p>
<p>然后，maven依赖添加：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">properties</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">java.version</span>&gt;</span>1.8<span class="tag">&lt;/<span class="name">java.version</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">project.build.sourceEncoding</span>&gt;</span>UTF-8<span class="tag">&lt;/<span class="name">project.build.sourceEncoding</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">corenlp.version</span>&gt;</span>3.6.0<span class="tag">&lt;/<span class="name">corenlp.version</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">properties</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">dependencies</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>edu.stanford.nlp<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>stanford-corenlp<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">version</span>&gt;</span>$&#123;corenlp.version&#125;<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>edu.stanford.nlp<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>stanford-corenlp<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">version</span>&gt;</span>$&#123;corenlp.version&#125;<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">classifier</span>&gt;</span>models<span class="tag">&lt;/<span class="name">classifier</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">dependency</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">groupId</span>&gt;</span>edu.stanford.nlp<span class="tag">&lt;/<span class="name">groupId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">artifactId</span>&gt;</span>stanford-corenlp<span class="tag">&lt;/<span class="name">artifactId</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">version</span>&gt;</span>$&#123;corenlp.version&#125;<span class="tag">&lt;/<span class="name">version</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">classifier</span>&gt;</span>models-chinese<span class="tag">&lt;/<span class="name">classifier</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">dependency</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">dependencies</span>&gt;</span></span><br></pre></td></tr></table></figure>
<p>测试：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> java.util.Properties;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> edu.stanford.nlp.ie.crf.CRFClassifier;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">CoreNLPSegment</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">private</span> <span class="keyword">static</span> CoreNLPSegment instance;</span><br><span class="line">    <span class="keyword">private</span> CRFClassifier         classifier;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">private</span> <span class="title">CoreNLPSegment</span><span class="params">()</span></span>&#123;</span><br><span class="line">        Properties props = <span class="keyword">new</span> Properties();</span><br><span class="line">        props.setProperty(<span class="string">"sighanCorporaDict"</span>, <span class="string">"data"</span>);</span><br><span class="line">        props.setProperty(<span class="string">"serDictionary"</span>, <span class="string">"data/dict-chris6.ser.gz"</span>);</span><br><span class="line">        props.setProperty(<span class="string">"inputEncoding"</span>, <span class="string">"UTF-8"</span>);</span><br><span class="line">        props.setProperty(<span class="string">"sighanPostProcessing"</span>, <span class="string">"true"</span>);</span><br><span class="line">        classifier = <span class="keyword">new</span> CRFClassifier(props);</span><br><span class="line">        classifier.loadClassifierNoExceptions(<span class="string">"data/ctb.gz"</span>, props);</span><br><span class="line">        classifier.flags.setProperties(props);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> CoreNLPSegment <span class="title">getInstance</span><span class="params">()</span> </span>&#123;</span><br><span class="line">        <span class="keyword">if</span> (instance == <span class="keyword">null</span>) &#123;</span><br><span class="line">            instance = <span class="keyword">new</span> CoreNLPSegment();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> instance;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">public</span> String[] doSegment(String data) &#123;</span><br><span class="line">        <span class="keyword">return</span> (String[]) classifier.segmentString(data).toArray();</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">        String sentence = <span class="string">"他和我在学校里常打桌球。"</span>;</span><br><span class="line">        String ret[] = CoreNLPSegment.getInstance().doSegment(sentence);</span><br><span class="line">        <span class="keyword">for</span> (String str : ret) &#123;</span><br><span class="line">            System.out.println(str);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p><strong>博客</strong>：</p>
<p><a href="https://blog.sectong.com/blog/corenlp_segment.html">https://blog.sectong.com/blog/corenlp_segment.html</a></p>
<p><a href="http://blog.csdn.net/lightty/article/details/51766602">http://blog.csdn.net/lightty/article/details/51766602</a></p>
<h3 id="12、Smartcn"><a href="#12、Smartcn" class="headerlink" title="12、Smartcn"></a>12、Smartcn</h3><p>Smartcn为Apache2.0协议的开源中文分词系统，Java语言编写，修改的中科院计算所ICTCLAS分词系统。很早以前看到Lucene上多了一个中文分词的contribution，当时只是简单的扫了一下.class文件的文件名，通过文件名可以看得出又是一个改的ICTCLAS的分词系统。</p>
<p><a href="http://lucene.apache.org/core/5_1_0/analyzers-smartcn/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.html">http://lucene.apache.org/core/5_1_0/analyzers-smartcn/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.html</a></p>
<h3 id="13、pinyin-分词器"><a href="#13、pinyin-分词器" class="headerlink" title="13、pinyin 分词器"></a>13、pinyin 分词器</h3><p>pinyin分词器可以让用户输入拼音，就能查找到相关的关键词。比如在某个商城搜索中，输入 <code>yonghui</code>，就能匹配到  <code>永辉</code>。这样的体验还是非常好的。</p>
<p>pinyin分词器的安装与IK是一样的。下载地址：<a href="https://github.com/medcl/elasticsearch-analysis-pinyin">https://github.com/medcl/elasticsearch-analysis-pinyin</a></p>
<p>一些参数请参考 GitHub 的 readme 文档。</p>
<p>这个分词器在1.8版本中，提供了两种分词规则：</p>
<ul>
<li><p>pinyin,就是普通的把汉字转换成拼音；</p>
</li>
<li><p>pinyin_first_letter，提取汉字的拼音首字母</p>
</li>
</ul>
<p>使用：</p>
<p>1.Create a index with custom pinyin analyzer</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">curl -XPUT http://localhost:9200/medcl/ -d&apos;</span><br><span class="line">&#123;</span><br><span class="line">    &quot;index&quot; : &#123;</span><br><span class="line">        &quot;analysis&quot; : &#123;</span><br><span class="line">            &quot;analyzer&quot; : &#123;</span><br><span class="line">                &quot;pinyin_analyzer&quot; : &#123;</span><br><span class="line">                    &quot;tokenizer&quot; : &quot;my_pinyin&quot;</span><br><span class="line">                    &#125;</span><br><span class="line">            &#125;,</span><br><span class="line">            &quot;tokenizer&quot; : &#123;</span><br><span class="line">                &quot;my_pinyin&quot; : &#123;</span><br><span class="line">                    &quot;type&quot; : &quot;pinyin&quot;,</span><br><span class="line">                    &quot;keep_separate_first_letter&quot; : false,</span><br><span class="line">                    &quot;keep_full_pinyin&quot; : true,</span><br><span class="line">                    &quot;keep_original&quot; : true,</span><br><span class="line">                    &quot;limit_first_letter_length&quot; : 16,</span><br><span class="line">                    &quot;lowercase&quot; : true,</span><br><span class="line">                    &quot;remove_duplicated_term&quot; : true</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;&apos;</span><br></pre></td></tr></table></figure>
<p>2.Test Analyzer, analyzing a chinese name, such as 刘德华</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">http://localhost:9200/medcl/_analyze?text=%e5%88%98%e5%be%b7%e5%8d%8e&amp;analyzer=pinyin_analyzer</span><br></pre></td></tr></table></figure>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;liu&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 0,</span><br><span class="line">      &quot;end_offset&quot; : 1,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 0</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;de&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 1,</span><br><span class="line">      &quot;end_offset&quot; : 2,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 1</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;hua&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 2,</span><br><span class="line">      &quot;end_offset&quot; : 3,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 2</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;刘德华&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 0,</span><br><span class="line">      &quot;end_offset&quot; : 3,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 3</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;ldh&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 0,</span><br><span class="line">      &quot;end_offset&quot; : 3,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 4</span><br><span class="line">    &#125;</span><br><span class="line">  ]</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>3.Create mapping</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/medcl/folks/_mapping -d&apos;</span><br><span class="line">&#123;</span><br><span class="line">    &quot;folks&quot;: &#123;</span><br><span class="line">        &quot;properties&quot;: &#123;</span><br><span class="line">            &quot;name&quot;: &#123;</span><br><span class="line">                &quot;type&quot;: &quot;keyword&quot;,</span><br><span class="line">                &quot;fields&quot;: &#123;</span><br><span class="line">                    &quot;pinyin&quot;: &#123;</span><br><span class="line">                        &quot;type&quot;: &quot;text&quot;,</span><br><span class="line">                        &quot;store&quot;: &quot;no&quot;,</span><br><span class="line">                        &quot;term_vector&quot;: &quot;with_offsets&quot;,</span><br><span class="line">                        &quot;analyzer&quot;: &quot;pinyin_analyzer&quot;,</span><br><span class="line">                        &quot;boost&quot;: 10</span><br><span class="line">                    &#125;</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;&apos;</span><br></pre></td></tr></table></figure>
<p>4.Indexing</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/medcl/folks/andy -d&apos;&#123;&quot;name&quot;:&quot;刘德华&quot;&#125;&apos;</span><br></pre></td></tr></table></figure>
<p>5.Let’s search</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">http://localhost:9200/medcl/folks/_search?q=name:%E5%88%98%E5%BE%B7%E5%8D%8E</span><br><span class="line">curl http://localhost:9200/medcl/folks/_search?q=name.pinyin:%e5%88%98%e5%be%b7</span><br><span class="line">curl http://localhost:9200/medcl/folks/_search?q=name.pinyin:liu</span><br><span class="line">curl http://localhost:9200/medcl/folks/_search?q=name.pinyin:ldh</span><br><span class="line">curl http://localhost:9200/medcl/folks/_search?q=name.pinyin:de+hua</span><br></pre></td></tr></table></figure>
<p>6.Using Pinyin-TokenFilter</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line">curl -XPUT http://localhost:9200/medcl1/ -d&apos;</span><br><span class="line">&#123;</span><br><span class="line">    &quot;index&quot; : &#123;</span><br><span class="line">        &quot;analysis&quot; : &#123;</span><br><span class="line">            &quot;analyzer&quot; : &#123;</span><br><span class="line">                &quot;user_name_analyzer&quot; : &#123;</span><br><span class="line">                    &quot;tokenizer&quot; : &quot;whitespace&quot;,</span><br><span class="line">                    &quot;filter&quot; : &quot;pinyin_first_letter_and_full_pinyin_filter&quot;</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;,</span><br><span class="line">            &quot;filter&quot; : &#123;</span><br><span class="line">                &quot;pinyin_first_letter_and_full_pinyin_filter&quot; : &#123;</span><br><span class="line">                    &quot;type&quot; : &quot;pinyin&quot;,</span><br><span class="line">                    &quot;keep_first_letter&quot; : true,</span><br><span class="line">                    &quot;keep_full_pinyin&quot; : false,</span><br><span class="line">                    &quot;keep_none_chinese&quot; : true,</span><br><span class="line">                    &quot;keep_original&quot; : false,</span><br><span class="line">                    &quot;limit_first_letter_length&quot; : 16,</span><br><span class="line">                    &quot;lowercase&quot; : true,</span><br><span class="line">                    &quot;trim_whitespace&quot; : true,</span><br><span class="line">                    &quot;keep_none_chinese_in_first_letter&quot; : true</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;&apos;</span><br></pre></td></tr></table></figure>
<p>Token Test:刘德华 张学友 郭富城 黎明 四大天王</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -XGET http://localhost:9200/medcl1/_analyze?text=%e5%88%98%e5%be%b7%e5%8d%8e+%e5%bc%a0%e5%ad%a6%e5%8f%8b+%e9%83%ad%e5%af%8c%e5%9f%8e+%e9%bb%8e%e6%98%8e+%e5%9b%9b%e5%a4%a7%e5%a4%a9%e7%8e%8b&amp;analyzer=user_name_analyzer</span><br></pre></td></tr></table></figure>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;ldh&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 0,</span><br><span class="line">      &quot;end_offset&quot; : 3,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 0</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;zxy&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 4,</span><br><span class="line">      &quot;end_offset&quot; : 7,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 1</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;gfc&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 8,</span><br><span class="line">      &quot;end_offset&quot; : 11,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 2</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;lm&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 12,</span><br><span class="line">      &quot;end_offset&quot; : 14,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 3</span><br><span class="line">    &#125;,</span><br><span class="line">    &#123;</span><br><span class="line">      &quot;token&quot; : &quot;sdtw&quot;,</span><br><span class="line">      &quot;start_offset&quot; : 15,</span><br><span class="line">      &quot;end_offset&quot; : 19,</span><br><span class="line">      &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">      &quot;position&quot; : 4</span><br><span class="line">    &#125;</span><br><span class="line">  ]</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>7.Used in phrase query</p>
<p>(1)、</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line">PUT /medcl/</span><br><span class="line"> &#123;</span><br><span class="line">     &quot;index&quot; : &#123;</span><br><span class="line">         &quot;analysis&quot; : &#123;</span><br><span class="line">             &quot;analyzer&quot; : &#123;</span><br><span class="line">                 &quot;pinyin_analyzer&quot; : &#123;</span><br><span class="line">                     &quot;tokenizer&quot; : &quot;my_pinyin&quot;</span><br><span class="line">                     &#125;</span><br><span class="line">             &#125;,</span><br><span class="line">             &quot;tokenizer&quot; : &#123;</span><br><span class="line">                 &quot;my_pinyin&quot; : &#123;</span><br><span class="line">                     &quot;type&quot; : &quot;pinyin&quot;,</span><br><span class="line">                     &quot;keep_first_letter&quot;:false,</span><br><span class="line">                     &quot;keep_separate_first_letter&quot; : false,</span><br><span class="line">                     &quot;keep_full_pinyin&quot; : true,</span><br><span class="line">                     &quot;keep_original&quot; : false,</span><br><span class="line">                     &quot;limit_first_letter_length&quot; : 16,</span><br><span class="line">                     &quot;lowercase&quot; : true</span><br><span class="line">                 &#125;</span><br><span class="line">             &#125;</span><br><span class="line">         &#125;</span><br><span class="line">     &#125;</span><br><span class="line"> &#125;</span><br><span class="line"> GET /medcl/folks/_search</span><br><span class="line"> &#123;</span><br><span class="line">   &quot;query&quot;: &#123;&quot;match_phrase&quot;: &#123;</span><br><span class="line">     &quot;name.pinyin&quot;: &quot;刘德华&quot;</span><br><span class="line">   &#125;&#125;</span><br><span class="line"> &#125;</span><br></pre></td></tr></table></figure>
<p>(2)、</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre></td><td class="code"><pre><span class="line">PUT /medcl/</span><br><span class="line">  &#123;</span><br><span class="line">      &quot;index&quot; : &#123;</span><br><span class="line">          &quot;analysis&quot; : &#123;</span><br><span class="line">              &quot;analyzer&quot; : &#123;</span><br><span class="line">                  &quot;pinyin_analyzer&quot; : &#123;</span><br><span class="line">                      &quot;tokenizer&quot; : &quot;my_pinyin&quot;</span><br><span class="line">                      &#125;</span><br><span class="line">              &#125;,</span><br><span class="line">              &quot;tokenizer&quot; : &#123;</span><br><span class="line">                  &quot;my_pinyin&quot; : &#123;</span><br><span class="line">                      &quot;type&quot; : &quot;pinyin&quot;,</span><br><span class="line">                      &quot;keep_first_letter&quot;:false,</span><br><span class="line">                      &quot;keep_separate_first_letter&quot; : true,</span><br><span class="line">                      &quot;keep_full_pinyin&quot; : false,</span><br><span class="line">                      &quot;keep_original&quot; : false,</span><br><span class="line">                      &quot;limit_first_letter_length&quot; : 16,</span><br><span class="line">                      &quot;lowercase&quot; : true</span><br><span class="line">                  &#125;</span><br><span class="line">              &#125;</span><br><span class="line">          &#125;</span><br><span class="line">      &#125;</span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">  POST /medcl/folks/andy</span><br><span class="line">  &#123;&quot;name&quot;:&quot;刘德华&quot;&#125;</span><br><span class="line"></span><br><span class="line">  GET /medcl/folks/_search</span><br><span class="line">  &#123;</span><br><span class="line">    &quot;query&quot;: &#123;&quot;match_phrase&quot;: &#123;</span><br><span class="line">      &quot;name.pinyin&quot;: &quot;刘德h&quot;</span><br><span class="line">    &#125;&#125;</span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">  GET /medcl/folks/_search</span><br><span class="line">  &#123;</span><br><span class="line">    &quot;query&quot;: &#123;&quot;match_phrase&quot;: &#123;</span><br><span class="line">      &quot;name.pinyin&quot;: &quot;刘dh&quot;</span><br><span class="line">    &#125;&#125;</span><br><span class="line">  &#125;</span><br><span class="line"></span><br><span class="line">  GET /medcl/folks/_search</span><br><span class="line">  &#123;</span><br><span class="line">    &quot;query&quot;: &#123;&quot;match_phrase&quot;: &#123;</span><br><span class="line">      &quot;name.pinyin&quot;: &quot;dh&quot;</span><br><span class="line">    &#125;&#125;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
<h3 id="14、Mmseg-分词器"><a href="#14、Mmseg-分词器" class="headerlink" title="14、Mmseg 分词器"></a>14、Mmseg 分词器</h3><p>也支持 Elasticsearch</p>
<p>下载地址：<a href="https://github.com/medcl/elasticsearch-analysis-mmseg/releases">https://github.com/medcl/elasticsearch-analysis-mmseg/releases</a>   根据对应的版本进行下载</p>
<p>如何使用：</p>
<p>1、创建索引：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -XPUT http://localhost:9200/index</span><br></pre></td></tr></table></figure>
<p>2、创建 mapping</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/index/fulltext/_mapping -d&apos;</span><br><span class="line">&#123;</span><br><span class="line">        &quot;properties&quot;: &#123;</span><br><span class="line">            &quot;content&quot;: &#123;</span><br><span class="line">                &quot;type&quot;: &quot;text&quot;,</span><br><span class="line">                &quot;term_vector&quot;: &quot;with_positions_offsets&quot;,</span><br><span class="line">                &quot;analyzer&quot;: &quot;mmseg_maxword&quot;,</span><br><span class="line">                &quot;search_analyzer&quot;: &quot;mmseg_maxword&quot;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">&#125;&apos;</span><br></pre></td></tr></table></figure>
<p>3.Indexing some docs</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/index/fulltext/1 -d&apos;</span><br><span class="line">&#123;&quot;content&quot;:&quot;美国留给伊拉克的是个烂摊子吗&quot;&#125;</span><br><span class="line">&apos;</span><br><span class="line"></span><br><span class="line">curl -XPOST http://localhost:9200/index/fulltext/2 -d&apos;</span><br><span class="line">&#123;&quot;content&quot;:&quot;公安部：各地校车将享最高路权&quot;&#125;</span><br><span class="line">&apos;</span><br><span class="line"></span><br><span class="line">curl -XPOST http://localhost:9200/index/fulltext/3 -d&apos;</span><br><span class="line">&#123;&quot;content&quot;:&quot;中韩渔警冲突调查：韩警平均每天扣1艘中国渔船&quot;&#125;</span><br><span class="line">&apos;</span><br><span class="line"></span><br><span class="line">curl -XPOST http://localhost:9200/index/fulltext/4 -d&apos;</span><br><span class="line">&#123;&quot;content&quot;:&quot;中国驻洛杉矶领事馆遭亚裔男子枪击 嫌犯已自首&quot;&#125;</span><br><span class="line">&apos;</span><br></pre></td></tr></table></figure>
<p>4.Query with highlighting(查询高亮)</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">curl -XPOST http://localhost:9200/index/fulltext/_search  -d&apos;</span><br><span class="line">&#123;</span><br><span class="line">    &quot;query&quot; : &#123; &quot;term&quot; : &#123; &quot;content&quot; : &quot;中国&quot; &#125;&#125;,</span><br><span class="line">    &quot;highlight&quot; : &#123;</span><br><span class="line">        &quot;pre_tags&quot; : [&quot;&lt;tag1&gt;&quot;, &quot;&lt;tag2&gt;&quot;],</span><br><span class="line">        &quot;post_tags&quot; : [&quot;&lt;/tag1&gt;&quot;, &quot;&lt;/tag2&gt;&quot;],</span><br><span class="line">        &quot;fields&quot; : &#123;</span><br><span class="line">            &quot;content&quot; : &#123;&#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line">&apos;</span><br></pre></td></tr></table></figure>
<p>5、结果：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">    &quot;took&quot;: 14,</span><br><span class="line">    &quot;timed_out&quot;: false,</span><br><span class="line">    &quot;_shards&quot;: &#123;</span><br><span class="line">        &quot;total&quot;: 5,</span><br><span class="line">        &quot;successful&quot;: 5,</span><br><span class="line">        &quot;failed&quot;: 0</span><br><span class="line">    &#125;,</span><br><span class="line">    &quot;hits&quot;: &#123;</span><br><span class="line">        &quot;total&quot;: 2,</span><br><span class="line">        &quot;max_score&quot;: 2,</span><br><span class="line">        &quot;hits&quot;: [</span><br><span class="line">            &#123;</span><br><span class="line">                &quot;_index&quot;: &quot;index&quot;,</span><br><span class="line">                &quot;_type&quot;: &quot;fulltext&quot;,</span><br><span class="line">                &quot;_id&quot;: &quot;4&quot;,</span><br><span class="line">                &quot;_score&quot;: 2,</span><br><span class="line">                &quot;_source&quot;: &#123;</span><br><span class="line">                    &quot;content&quot;: &quot;中国驻洛杉矶领事馆遭亚裔男子枪击 嫌犯已自首&quot;</span><br><span class="line">                &#125;,</span><br><span class="line">                &quot;highlight&quot;: &#123;</span><br><span class="line">                    &quot;content&quot;: [</span><br><span class="line">                        &quot;&lt;tag1&gt;中国&lt;/tag1&gt;驻洛杉矶领事馆遭亚裔男子枪击 嫌犯已自首 &quot;</span><br><span class="line">                    ]</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;,</span><br><span class="line">            &#123;</span><br><span class="line">                &quot;_index&quot;: &quot;index&quot;,</span><br><span class="line">                &quot;_type&quot;: &quot;fulltext&quot;,</span><br><span class="line">                &quot;_id&quot;: &quot;3&quot;,</span><br><span class="line">                &quot;_score&quot;: 2,</span><br><span class="line">                &quot;_source&quot;: &#123;</span><br><span class="line">                    &quot;content&quot;: &quot;中韩渔警冲突调查：韩警平均每天扣1艘中国渔船&quot;</span><br><span class="line">                &#125;,</span><br><span class="line">                &quot;highlight&quot;: &#123;</span><br><span class="line">                    &quot;content&quot;: [</span><br><span class="line">                        &quot;均每天扣1艘&lt;tag1&gt;中国&lt;/tag1&gt;渔船 &quot;</span><br><span class="line">                    ]</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;</span><br><span class="line">        ]</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>参考博客：</p>
<p>为elastic添加中文分词: <a href="http://blog.csdn.net/dingzfang/article/details/42776693">http://blog.csdn.net/dingzfang/article/details/42776693</a></p>
<h3 id="15、bosonnlp-（玻森数据中文分析器）"><a href="#15、bosonnlp-（玻森数据中文分析器）" class="headerlink" title="15、bosonnlp （玻森数据中文分析器）"></a>15、bosonnlp （玻森数据中文分析器）</h3><p>下载地址：<a href="https://github.com/bosondata/elasticsearch-analysis-bosonnlp">https://github.com/bosondata/elasticsearch-analysis-bosonnlp</a></p>
<p>如何使用：</p>
<p>运行 ElasticSearch 之前需要在 config 文件夹中修改 elasticsearch.yml 来定义使用玻森中文分析器，并填写玻森 API_TOKEN 以及玻森分词 API 的地址，即在该文件结尾处添加：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">index:</span><br><span class="line">  analysis:</span><br><span class="line">    analyzer:</span><br><span class="line">      bosonnlp:</span><br><span class="line">          type: bosonnlp</span><br><span class="line">          API_URL: http://api.bosonnlp.com/tag/analysis</span><br><span class="line">          # You MUST give the API_TOKEN value, otherwise it doesn&apos;t work</span><br><span class="line">          API_TOKEN: *PUT YOUR API TOKEN HERE*</span><br><span class="line">          # Please uncomment if you want to specify ANY ONE of the following</span><br><span class="line">          # areguments, otherwise the DEFAULT value will be used, i.e.,</span><br><span class="line">          # space_mode is 0,</span><br><span class="line">          # oov_level is 3,</span><br><span class="line">          # t2s is 0,</span><br><span class="line">          # special_char_conv is 0.</span><br><span class="line">          # More detials can be found in bosonnlp docs:</span><br><span class="line">          # http://docs.bosonnlp.com/tag.html</span><br><span class="line">          #</span><br><span class="line">          #</span><br><span class="line">          # space_mode: put your value here(range from 0-3)</span><br><span class="line">          # oov_level: put your value here(range from 0-4)</span><br><span class="line">          # t2s: put your value here(range from 0-1)</span><br><span class="line">          # special_char_conv: put your value here(range from 0-1)</span><br></pre></td></tr></table></figure>
<p>需要注意的是</p>
<p>必须在 API_URL 填写给定的分词地址以及在API_TOKEN：<em>PUT YOUR API TOKEN HERE</em> 中填写给定的玻森数据API_TOKEN，否则无法使用玻森中文分析器。该 API_TOKEN 是注册玻森数据账号所获得。</p>
<p>如果配置文件中已经有配置过其他的 analyzer，请直接在 analyzer 下如上添加 bosonnlp analyzer。</p>
<p>如果有多个 node 并且都需要 BosonNLP 的分词插件，则每个 node 下的 yaml 文件都需要如上安装和设置。</p>
<p>另外，玻森中文分词还提供了4个参数（space_mode，oov_level，t2s，special_char_conv）可满足不同的分词需求。如果取默认值，则无需任何修改；否则，可取消对应参数的注释并赋值。</p>
<p><strong>测试：</strong></p>
<p>建立 index</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -XPUT &apos;localhost:9200/test&apos;</span><br></pre></td></tr></table></figure>
<p>测试分析器是否配置成功</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -XGET &apos;localhost:9200/test/_analyze?analyzer=bosonnlp&amp;pretty&apos; -d &apos;这是玻森数据分词的测试&apos;</span><br></pre></td></tr></table></figure>
<p>结果</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [ &#123;</span><br><span class="line">    &quot;token&quot; : &quot;这&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 0,</span><br><span class="line">    &quot;end_offset&quot; : 1,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 0</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;是&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 1,</span><br><span class="line">    &quot;end_offset&quot; : 2,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 1</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;玻森&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 2,</span><br><span class="line">    &quot;end_offset&quot; : 4,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 2</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;数据&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 4,</span><br><span class="line">    &quot;end_offset&quot; : 6,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 3</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;分词&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 6,</span><br><span class="line">    &quot;end_offset&quot; : 8,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 4</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;的&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 8,</span><br><span class="line">    &quot;end_offset&quot; : 9,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 5</span><br><span class="line">  &#125;, &#123;</span><br><span class="line">    &quot;token&quot; : &quot;测试&quot;,</span><br><span class="line">    &quot;start_offset&quot; : 9,</span><br><span class="line">    &quot;end_offset&quot; : 11,</span><br><span class="line">    &quot;type&quot; : &quot;word&quot;,</span><br><span class="line">    &quot;position&quot; : 6</span><br><span class="line">  &#125; ]</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>配置 Token Filter</p>
<p>现有的 BosonNLP 分析器没有内置 token filter，如果有过滤 Token 的需求，可以利用 BosonNLP Tokenizer 和 ES 提供的 token filter 搭建定制分析器。</p>
<p>步骤</p>
<p>配置定制的 analyzer 有以下三个步骤：</p>
<p>添加 BosonNLP tokenizer<br>在 elasticsearch.yml 文件中 analysis 下添加 tokenizer， 并在 tokenizer 中添加 BosonNLP tokenizer 的配置：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line">index:</span><br><span class="line">  analysis:</span><br><span class="line">    analyzer:</span><br><span class="line">      ...</span><br><span class="line">    tokenizer:</span><br><span class="line">      bosonnlp:</span><br><span class="line">          type: bosonnlp</span><br><span class="line">          API_URL: http://api.bosonnlp.com/tag/analysis</span><br><span class="line">          # You MUST give the API_TOKEN value, otherwise it doesn&apos;t work</span><br><span class="line">          API_TOKEN: *PUT YOUR API TOKEN HERE*</span><br><span class="line">          # Please uncomment if you want to specify ANY ONE of the following</span><br><span class="line">          # areguments, otherwise the DEFAULT value will be used, i.e.,</span><br><span class="line">          # space_mode is 0,</span><br><span class="line">          # oov_level is 3,</span><br><span class="line">          # t2s is 0,</span><br><span class="line">          # special_char_conv is 0.</span><br><span class="line">          # More detials can be found in bosonnlp docs:</span><br><span class="line">          # http://docs.bosonnlp.com/tag.html</span><br><span class="line">          #</span><br><span class="line">          #</span><br><span class="line">          # space_mode: put your value here(range from 0-3)</span><br><span class="line">          # oov_level: put your value here(range from 0-4)</span><br><span class="line">          # t2s: put your value here(range from 0-1)</span><br><span class="line">          # special_char_conv: put your value here(range from 0-1)</span><br></pre></td></tr></table></figure>
<p>添加 token filter</p>
<p>在 elasticsearch.yml 文件中 analysis 下添加 filter， 并在 filter 中添加所需 filter 的配置（下面例子中，我们以 lowercase filter 为例）：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">index:</span><br><span class="line">  analysis:</span><br><span class="line">    analyzer:</span><br><span class="line">      ...</span><br><span class="line">    tokenizer:</span><br><span class="line">      ...</span><br><span class="line">    filter:</span><br><span class="line">      lowercase:</span><br><span class="line">          type: lowercase</span><br></pre></td></tr></table></figure>
<p>添加定制的 analyzer</p>
<p>在 elasticsearch.yml 文件中 analysis 下添加 analyzer， 并在 analyzer 中添加定制的 analyzer 的配置（下面例子中，我们把定制的 analyzer 命名为 filter_bosonnlp）：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">index:</span><br><span class="line">  analysis:</span><br><span class="line">    analyzer:</span><br><span class="line">      ...</span><br><span class="line">      filter_bosonnlp:</span><br><span class="line">          type: custom</span><br><span class="line">          tokenizer: bosonnlp</span><br><span class="line">          filter: [lowercase]</span><br></pre></td></tr></table></figure>
<hr>
<h2 id="自定义分词器"><a href="#自定义分词器" class="headerlink" title="自定义分词器"></a>自定义分词器</h2><p>虽然Elasticsearch带有一些现成的分析器，然而在分析器上Elasticsearch真正的强大之处在于，你可以通过在一个适合你的特定数据的设置之中组合字符过滤器、分词器、词汇单元过滤器来创建自定义的分析器。</p>
<p><strong>字符过滤器</strong>：</p>
<p>字符过滤器 用来 整理 一个尚未被分词的字符串。例如，如果我们的文本是HTML格式的，它会包含像 <code>&lt;p&gt;</code> 或者 <code>&lt;div&gt;</code> 这样的HTML标签，这些标签是我们不想索引的。我们可以使用 html清除 字符过滤器 来移除掉所有的HTML标签，并且像把 <code>&amp;Aacute;</code> 转换为相对应的Unicode字符 Á 这样，转换HTML实体。</p>
<p>一个分析器可能有0个或者多个字符过滤器。</p>
<p><strong>分词器</strong>:</p>
<p>一个分析器 必须 有一个唯一的分词器。 分词器把字符串分解成单个词条或者词汇单元。 标准 分析器里使用的 标准 分词器 把一个字符串根据单词边界分解成单个词条，并且移除掉大部分的标点符号，然而还有其他不同行为的分词器存在。</p>
<p><strong>词单元过滤器</strong>:</p>
<p>经过分词，作为结果的 词单元流 会按照指定的顺序通过指定的词单元过滤器 。</p>
<p>词单元过滤器可以修改、添加或者移除词单元。我们已经提到过 lowercase 和 stop 词过滤器 ，但是在 Elasticsearch 里面还有很多可供选择的词单元过滤器。 词干过滤器 把单词 遏制 为 词干。 ascii_folding 过滤器移除变音符，把一个像 “très” 这样的词转换为 “tres” 。 ngram 和 edge_ngram 词单元过滤器 可以产生 适合用于部分匹配或者自动补全的词单元。</p>
<h3 id="创建一个自定义分析器"><a href="#创建一个自定义分析器" class="headerlink" title="创建一个自定义分析器"></a>创建一个自定义分析器</h3><p>我们可以在 analysis 下的相应位置设置字符过滤器、分词器和词单元过滤器:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">PUT /my_index</span><br><span class="line">&#123;</span><br><span class="line">    &quot;settings&quot;: &#123;</span><br><span class="line">        &quot;analysis&quot;: &#123;</span><br><span class="line">            &quot;char_filter&quot;: &#123; ... custom character filters ... &#125;,</span><br><span class="line">            &quot;tokenizer&quot;:   &#123; ...    custom tokenizers     ... &#125;,</span><br><span class="line">            &quot;filter&quot;:      &#123; ...   custom token filters   ... &#125;,</span><br><span class="line">            &quot;analyzer&quot;:    &#123; ...    custom analyzers      ... &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>这个分析器可以做到下面的这些事:</p>
<p>1、使用 html清除 字符过滤器移除HTML部分。</p>
<p>2、使用一个自定义的 映射 字符过滤器把 &amp; 替换为 “和” ：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">&quot;char_filter&quot;: &#123;</span><br><span class="line">    &quot;&amp;_to_and&quot;: &#123;</span><br><span class="line">        &quot;type&quot;:       &quot;mapping&quot;,</span><br><span class="line">        &quot;mappings&quot;: [ &quot;&amp;=&gt; and &quot;]</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>3、使用 标准 分词器分词。</p>
<p>4、小写词条，使用 小写 词过滤器处理。</p>
<p>5、使用自定义 停止 词过滤器移除自定义的停止词列表中包含的词：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">&quot;filter&quot;: &#123;</span><br><span class="line">    &quot;my_stopwords&quot;: &#123;</span><br><span class="line">        &quot;type&quot;:        &quot;stop&quot;,</span><br><span class="line">        &quot;stopwords&quot;: [ &quot;the&quot;, &quot;a&quot; ]</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>我们的分析器定义用我们之前已经设置好的自定义过滤器组合了已经定义好的分词器和过滤器：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">&quot;analyzer&quot;: &#123;</span><br><span class="line">    &quot;my_analyzer&quot;: &#123;</span><br><span class="line">        &quot;type&quot;:           &quot;custom&quot;,</span><br><span class="line">        &quot;char_filter&quot;:  [ &quot;html_strip&quot;, &quot;&amp;_to_and&quot; ],</span><br><span class="line">        &quot;tokenizer&quot;:      &quot;standard&quot;,</span><br><span class="line">        &quot;filter&quot;:       [ &quot;lowercase&quot;, &quot;my_stopwords&quot; ]</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>汇总起来，完整的 创建索引 请求 看起来应该像这样：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line">PUT /my_index</span><br><span class="line">&#123;</span><br><span class="line">    &quot;settings&quot;: &#123;</span><br><span class="line">        &quot;analysis&quot;: &#123;</span><br><span class="line">            &quot;char_filter&quot;: &#123;</span><br><span class="line">                &quot;&amp;_to_and&quot;: &#123;</span><br><span class="line">                    &quot;type&quot;:       &quot;mapping&quot;,</span><br><span class="line">                    &quot;mappings&quot;: [ &quot;&amp;=&gt; and &quot;]</span><br><span class="line">            &#125;&#125;,</span><br><span class="line">            &quot;filter&quot;: &#123;</span><br><span class="line">                &quot;my_stopwords&quot;: &#123;</span><br><span class="line">                    &quot;type&quot;:       &quot;stop&quot;,</span><br><span class="line">                    &quot;stopwords&quot;: [ &quot;the&quot;, &quot;a&quot; ]</span><br><span class="line">            &#125;&#125;,</span><br><span class="line">            &quot;analyzer&quot;: &#123;</span><br><span class="line">                &quot;my_analyzer&quot;: &#123;</span><br><span class="line">                    &quot;type&quot;:         &quot;custom&quot;,</span><br><span class="line">                    &quot;char_filter&quot;:  [ &quot;html_strip&quot;, &quot;&amp;_to_and&quot; ],</span><br><span class="line">                    &quot;tokenizer&quot;:    &quot;standard&quot;,</span><br><span class="line">                    &quot;filter&quot;:       [ &quot;lowercase&quot;, &quot;my_stopwords&quot; ]</span><br><span class="line">            &#125;&#125;</span><br><span class="line">&#125;&#125;&#125;</span><br></pre></td></tr></table></figure>
<p>索引被创建以后，使用 analyze API 来 测试这个新的分析器：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">GET /my_index/_analyze?analyzer=my_analyzer</span><br><span class="line">The quick &amp; brown fox</span><br></pre></td></tr></table></figure>
<p>下面的缩略结果展示出我们的分析器正在正确地运行：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">  &quot;tokens&quot; : [</span><br><span class="line">      &#123; &quot;token&quot; :   &quot;quick&quot;,    &quot;position&quot; : 2 &#125;,</span><br><span class="line">      &#123; &quot;token&quot; :   &quot;and&quot;,      &quot;position&quot; : 3 &#125;,</span><br><span class="line">      &#123; &quot;token&quot; :   &quot;brown&quot;,    &quot;position&quot; : 4 &#125;,</span><br><span class="line">      &#123; &quot;token&quot; :   &quot;fox&quot;,      &quot;position&quot; : 5 &#125;</span><br><span class="line">    ]</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>这个分析器现在是没有多大用处的，除非我们告诉 Elasticsearch在哪里用上它。我们可以像下面这样把这个分析器应用在一个 string 字段上：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">PUT /my_index/_mapping/my_type</span><br><span class="line">&#123;</span><br><span class="line">    &quot;properties&quot;: &#123;</span><br><span class="line">        &quot;title&quot;: &#123;</span><br><span class="line">            &quot;type&quot;:      &quot;string&quot;,</span><br><span class="line">            &quot;analyzer&quot;:  &quot;my_analyzer&quot;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="最后"><a href="#最后" class="headerlink" title="最后"></a>最后</h3><p>整理参考网上资料，如有不正确的地方还请多多指教！</p>
<p><img src="https://ws3.sinaimg.cn/large/006tNc79gy1fp3jkmizmpj30o00didgn.jpg" alt=""></p>

      
      
        <div class="page-reward">
          <p><a href="javascript:void(0)" onclick="dashangToggle()" class="dashang">赏</a></p>
          <div class="hide_box"></div>
          <div class="shang_box">
            <a class="shang_close" href="javascript:void(0)" onclick="dashangToggle()">×</a>
            <div class="shang_tit">
              <p>纯属好玩</p>
            </div>
            <div class="shang_payimg">
              <img src="/img/alipayimg.jpg" alt="扫码支持" title="扫一扫" />
            </div>
              <div class="pay_explain">扫码打赏，你说多少就多少</div>
            <div class="shang_payselect">
              
                <div class="pay_item checked" data-id="alipay">
                  <span class="radiobox"></span>
                  <span class="pay_logo"><img src="/img/alipay.png" alt="支付宝" /></span>
                </div>
              
              
                <div class="pay_item" data-id="wechat">
                  <span class="radiobox"></span>
                  <span class="pay_logo"><img src="/img/weixin.png" alt="微信" /></span>
                </div>
              
            </div>
            <div class="shang_info">
              <p>打开<span id="shang_pay_txt">支付宝</span>扫一扫，即可进行扫码打赏哦</p>
            </div>
          </div>
        </div>
        <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/zepto/1.2.0/zepto.min.js"></script>
        <script type="text/javascript">
          $(".pay_item").click(function(){
            $(this).addClass('checked').siblings('.pay_item').removeClass('checked');
            var dataid=$(this).attr('data-id');
            $(".shang_payimg img").attr("src","/img/"+dataid+"img.jpg");
            $("#shang_pay_txt").text(dataid=="alipay"?"支付宝":"微信");
          });
          function dashangToggle(){
            
            $(".hide_box").fadeToggle();
            $(".shang_box").fadeToggle();
          }
        </script>
      
    </div>
    
  </div>
  
    
    <div class="copyright">
        <p><span>本文标题:</span><a href="/2017/09/08/Elasticsearch-analyzers/">Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法</a></p>
        <p><span>文章作者:</span><a href="/" title="访问 zhisheng 的个人博客">zhisheng</a></p>
        <p><span>发布时间:</span>2017年09月08日 - 00时00分</p>
        <p><span>最后更新:</span>2018年03月24日 - 10时25分</p>
        <p>
            <span>原始链接:</span><a class="post-url" href="/2017/09/08/Elasticsearch-analyzers/" title="Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法">http://www.54tianzhisheng.cn/2017/09/08/Elasticsearch-analyzers/</a>
            <span class="copy-path" data-clipboard-text="原文: http://www.54tianzhisheng.cn/2017/09/08/Elasticsearch-analyzers/　　作者: zhisheng" title="点击复制文章链接"><i class="fa fa-clipboard"></i></span>
            <script src="/js/clipboard.min.js"></script>
            <script> var clipboard = new Clipboard('.copy-path'); </script>
        </p>
        <p>
            <span>许可协议:</span><i class="fa fa-creative-commons"></i> <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/3.0/cn/" title="中国大陆 (CC BY-NC-SA 3.0 CN)" target = "_blank">"署名-非商用-相同方式共享 3.0"</a> 转载请保留原文链接及作者。
        </p>
    </div>



<nav id="article-nav">
  
    <a href="/2017/09/09/Elasticsearch-install/" id="article-nav-newer" class="article-nav-link-wrap">
      <strong class="article-nav-caption"><</strong>
      <div class="article-nav-title">
        
          Elasticsearch 系列文章（二）：全文搜索引擎 Elasticsearch 集群搭建入门教程
        
      </div>
    </a>
  
  
    <a href="/2017/08/28/recommend-books/" id="article-nav-older" class="article-nav-link-wrap">
      <div class="article-nav-title">那些年我看过的书 —— 致敬我的大学生活 —— Say Good Bye ！</div>
      <strong class="article-nav-caption">></strong>
    </a>
  
</nav>

  
</article>

    <div id="toc" class="toc-article">
    <strong class="toc-title">文章目录</strong>
    <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#系统默认分词器："><span class="toc-number">1.</span> <span class="toc-text">系统默认分词器：</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1、standard-分词器"><span class="toc-number">1.1.</span> <span class="toc-text">1、standard 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2、simple-分词器"><span class="toc-number">1.2.</span> <span class="toc-text">2、simple 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3、Whitespace-分词器"><span class="toc-number">1.3.</span> <span class="toc-text">3、Whitespace 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#4、Stop-分词器"><span class="toc-number">1.4.</span> <span class="toc-text">4、Stop 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5、keyword-分词器"><span class="toc-number">1.5.</span> <span class="toc-text">5、keyword 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#6、pattern-分词器"><span class="toc-number">1.6.</span> <span class="toc-text">6、pattern 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#7、language-分词器"><span class="toc-number">1.7.</span> <span class="toc-text">7、language 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#8、snowball-分词器"><span class="toc-number">1.8.</span> <span class="toc-text">8、snowball 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#9、Custom-分词器"><span class="toc-number">1.9.</span> <span class="toc-text">9、Custom 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#10、fingerprint-分词器"><span class="toc-number">1.10.</span> <span class="toc-text">10、fingerprint 分词器</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#中文分词器："><span class="toc-number">2.</span> <span class="toc-text">中文分词器：</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1、ik-analyzer"><span class="toc-number">2.1.</span> <span class="toc-text">1、ik-analyzer</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#Elasticsearch添加中文分词"><span class="toc-number">2.1.1.</span> <span class="toc-text">Elasticsearch添加中文分词</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#安装IK分词插件"><span class="toc-number">2.1.1.1.</span> <span class="toc-text">安装IK分词插件</span></a></li></ol></li><li class="toc-item toc-level-4"><a class="toc-link" href="#ik-带有两个分词器"><span class="toc-number">2.1.2.</span> <span class="toc-text">ik 带有两个分词器</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#热词更新配置"><span class="toc-number">2.1.3.</span> <span class="toc-text">热词更新配置</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2、结巴中文分词"><span class="toc-number">2.2.</span> <span class="toc-text">2、结巴中文分词</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#特点："><span class="toc-number">2.2.1.</span> <span class="toc-text">特点：</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3、THULAC"><span class="toc-number">2.3.</span> <span class="toc-text">3、THULAC</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#4、NLPIR"><span class="toc-number">2.4.</span> <span class="toc-text">4、NLPIR</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5、ansj分词器"><span class="toc-number">2.5.</span> <span class="toc-text">5、ansj分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#6、哈工大的LTP"><span class="toc-number">2.6.</span> <span class="toc-text">6、哈工大的LTP</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#7、庖丁解牛"><span class="toc-number">2.7.</span> <span class="toc-text">7、庖丁解牛</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#8、sogo在线分词"><span class="toc-number">2.8.</span> <span class="toc-text">8、sogo在线分词</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#9、word分词"><span class="toc-number">2.9.</span> <span class="toc-text">9、word分词</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#10、jcseg分词器"><span class="toc-number">2.10.</span> <span class="toc-text">10、jcseg分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#11、stanford分词器"><span class="toc-number">2.11.</span> <span class="toc-text">11、stanford分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#12、Smartcn"><span class="toc-number">2.12.</span> <span class="toc-text">12、Smartcn</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#13、pinyin-分词器"><span class="toc-number">2.13.</span> <span class="toc-text">13、pinyin 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#14、Mmseg-分词器"><span class="toc-number">2.14.</span> <span class="toc-text">14、Mmseg 分词器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#15、bosonnlp-（玻森数据中文分析器）"><span class="toc-number">2.15.</span> <span class="toc-text">15、bosonnlp （玻森数据中文分析器）</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#自定义分词器"><span class="toc-number">3.</span> <span class="toc-text">自定义分词器</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#创建一个自定义分析器"><span class="toc-number">3.1.</span> <span class="toc-text">创建一个自定义分析器</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#最后"><span class="toc-number">3.2.</span> <span class="toc-text">最后</span></a></li></ol></li></ol>
</div>
<input type="button" id="tocButton" value="隐藏目录"  title="点击按钮隐藏或者显示文章目录">

<script src="https://7.url.cn/edu/jslib/comb/require-2.1.6,jquery-1.9.1.min.js"></script>
<script>
    var valueHide = "隐藏目录";
    var valueShow = "显示目录";

    if ($(".left-col").is(":hidden")) {
        $("#tocButton").attr("value", valueShow);
    }
    $("#tocButton").click(function() {
        if ($("#toc").is(":hidden")) {
            $("#tocButton").attr("value", valueHide);
            $("#toc").slideDown(320);
        }
        else {
            $("#tocButton").attr("value", valueShow);
            $("#toc").slideUp(350);
        }
    })
    if ($(".toc").length < 1) {
        $("#toc, #tocButton").hide();
    }
</script>





<div class="bdsharebuttonbox">
	<a href="#" class="fx fa-weibo bds_tsina" data-cmd="tsina" title="分享到新浪微博"></a>
	<a href="#" class="fx fa-weixin bds_weixin" data-cmd="weixin" title="分享到微信"></a>
	<a href="#" class="fx fa-qq bds_sqq" data-cmd="sqq" title="分享到QQ好友"></a>
	<a href="#" class="fx fa-facebook-official bds_fbook" data-cmd="fbook" title="分享到Facebook"></a>
	<a href="#" class="fx fa-twitter bds_twi" data-cmd="twi" title="分享到Twitter"></a>
	<a href="#" class="fx fa-linkedin bds_linkedin" data-cmd="linkedin" title="分享到linkedin"></a>
	<a href="#" class="fx fa-files-o bds_copy" data-cmd="copy" title="分享到复制网址"></a>
</div>
<script>window._bd_share_config={"common":{"bdSnsKey":{},"bdText":"","bdMini":"2","bdMiniList":false,"bdPic":"","bdStyle":"2","bdSize":"24"},"share":{}};with(document)0[(getElementsByTagName('head')[0]||body).appendChild(createElement('script')).src='/static/api/js/share.js?v=89860593.js?cdnversion='+~(-new Date()/36e5)];</script>




    
        <div id="gitments"></div>
<script src="/js/gitment.browser.js"></script>
<script>
    var gitment = new Gitment({
      id: window.location.pathname,
      owner: 'zhisheng',
      repo: 'zhisheng17.github.io',
      oauth: {
        client_id: '',
        client_secret: '',
      },
    })
    gitment.render('gitments')
</script>
    



    <div class="scroll" id="post-nav-button">
        
            <a href="/2017/09/09/Elasticsearch-install/" title="上一篇: Elasticsearch 系列文章（二）：全文搜索引擎 Elasticsearch 集群搭建入门教程">
                <i class="fa fa-angle-left"></i>
            </a>
        
        <a title="文章列表"><i class="fa fa-bars"></i><i class="fa fa-times"></i></a>
        
            <a href="/2017/08/28/recommend-books/" title="下一篇: 那些年我看过的书 —— 致敬我的大学生活 —— Say Good Bye ！">
                <i class="fa fa-angle-right"></i>
            </a>
        
    </div>
    <ul class="post-list"><li class="post-list-item"><a class="post-list-link" href="/2019/01/06/Flink-Kafka-sink/">《从0到1学习Flink》—— Flink 写入数据到 Kafka</a></li><li class="post-list-item"><a class="post-list-link" href="/2019/01/05/Flink-run/">《从0到1学习Flink》—— Flink 项目如何运行？</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/12/30/Flink-ElasticSearch-Sink/">《从0到1学习Flink》—— Flink 写入数据到 ElasticSearch</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/12/11/Flink-time/">《从0到1学习Flink》—— Flink 中几种 Time 详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/12/08/Flink-Stream-Windows/">《从0到1学习Flink》—— 介绍Flink中的Stream Windows</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/11/04/Flink-Data-transformation/">《从0到1学习Flink》—— Flink Data transformation(转换)</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/10/31/flink-create-sink/">《从0到1学习Flink》—— 如何自定义 Data Sink ？</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/10/30/flink-create-source/">《从0到1学习Flink》—— 如何自定义 Data Source ？</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/10/29/flink-sink/">《从0到1学习Flink》—— Data Sink 介绍</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/10/28/flink-sources/">《从0到1学习Flink》—— Data Source 介绍</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/10/27/flink-config/">《从0到1学习Flink》—— Flink 配置文件详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/10/13/flink-introduction/">《从0到1学习Flink》—— Apache Flink 介绍</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/09/18/flink-install/">《从0到1学习Flink》—— Mac 上搭建 Flink 1.6.0 环境并构建运行简单程序入门</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/08/30/go-sync/">Go 并发——实现协程同步的几种方式</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/08/14/idea-remote-debug-elasticsearch/">教你如何在 IDEA 远程 Debug ElasticSearch</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/08/12/es-code03/">渣渣菜鸡的 ElasticSearch 源码解析 —— 启动流程（下）</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/08/11/es-code02/">渣渣菜鸡的 ElasticSearch 源码解析 —— 启动流程（上）</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/08/05/es-code01/">渣渣菜鸡的 ElasticSearch 源码解析 —— 环境搭建</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/08/04/why-see-es-code/">渣渣菜鸡为什么要看 ElasticSearch 源码？</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/07/31/alipay02/">渣渣菜鸡的蚂蚁金服面试经历(二)</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/07/30/alipay01/">渣渣菜鸡的蚂蚁金服面试经历(一)</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/07/12/youzan/">渣渣菜鸡的有赞面试经历（一）</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/06/20/java-8-date/">20 个案例教你在 Java 8 中如何处理日期和时间?</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/06/19/SimpleDateFormat/">SimpleDateFormat 如何安全的使用？</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/05/26/paper/">苦逼的毕业论文经历</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/04/30/springboot_SpringApplication/">Spring Boot 2.0系列文章(七)：SpringApplication 深入探索</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/04/24/Distributed_lock/">分布式锁看这篇就够了</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/04/19/SpringBootApplication-annotation/">Spring Boot 2.0系列文章(六)：Spring Boot 2.0中SpringBootApplication注解详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/04/18/spring_boot2_project/">Spring Boot 2.0系列文章(五)：Spring Boot 2.0 项目源码结构预览</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/04/15/springboot2_code/">Spring Boot 2.0系列文章(四)：Spring Boot 2.0 源码阅读环境搭建</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/04/13/Spring_Boot_2.0_Configuration_Changelog/">Spring Boot 2.0系列文章(三)：Spring Boot 2.0 配置改变</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/03/27/blogs/">写这么多系列博客，怪不得找不到女朋友</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/03/06/SpringBoot2-new-features/">Spring Boot 2.0系列文章(二)：Spring Boot 2.0 新特性详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/03/06/SpringBoot2-Migration-Guide/">Spring Boot 2.0系列文章(一)：Spring Boot 2.0 迁移指南</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/03/04/springboot-vedio/">小马哥 《Java 微服务实践 - Spring Boot 系列》强烈推荐</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/03/04/springcloud-vedio/">小马哥 《Java 微服务实践 - Spring Cloud 系列》强烈推荐</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/02/28/Java-Memory-Model/">《深入理解 Java 内存模型》读书笔记</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/02/07/SpringBoot-RocketMQ/">Spring Boot系列文章（六）：SpringBoot RocketMQ 整合使用和监控</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/02/07/rocketmq-example/">RocketMQ系列文章（三）：RocketMQ 简单的消息示例</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/02/06/RocketMQ-install/">RocketMQ系列文章（二）：RocketMQ 安装及快速入门</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/02/05/RocketMQ/">RocketMQ系列文章（一）：RocketMQ 初探</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/28/RabbitMQ/">Spring Boot系列文章（五）：SpringBoot RabbitMQ 整合进阶版</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/27/SpringBoot-ActiveMQ/">Spring Boot系列文章（四）：SpringBoot ActiveMQ 整合使用</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/26/SpringBoot-RabbitMQ/">Spring Boot系列文章（三）：SpringBoot  RabbitMQ 整合使用</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/25/Docker-install/">Docker系列文章（二）：Mac 安装 Docker 及常用命令</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/24/mac/">MacBook Pro 初体验</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/17/SpringBoot-Admin/">Spring Boot系列文章（二）：SpringBoot Admin 使用指南</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/09/lombok/">Lombok 看这篇就够了</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/05/SpringBoot-Kafka/">Spring Boot系列文章（一）：SpringBoot Kafka 整合使用</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/04/Kafka/">Kafka 安装及快速入门</a></li><li class="post-list-item"><a class="post-list-link" href="/2018/01/04/weixin/">为什么要重新运营以前的公众号呢？</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/27/consul-install/">Windows 下安装 Consul</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/25/ELK/">Elasticsearch 系列文章（五）：ELK 实时日志分析平台环境搭建</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/18/hexo-yilia/">Hexo + yilia 搭建博客可能会遇到的所有疑问</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/13/Google-Developer-Days/">谷歌开发者大会收获满满，不去真 “可惜” 了</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/09/CodeMirror/">使用 CodeMirror 打造属于自己的在线代码编辑器</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/08/netty-01-env/">Netty系列文章（一）：Netty 源码阅读之初始环境搭建</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/03/RestTemplate/">RestTemplate 详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/12/02/wx-01/">实习圈群里提问小记</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/11/26/Docker-harbor/">Docker系列文章（一）：基于 Harbor 搭建 Docker 私有镜像仓库</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/11/18/flow-control/">基于分布式环境下限流系统的设计</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/11/18/Money-management/">谈谈我的理财</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/11/11/recommended-books/">送你一份双十一剁手书单【墙裂推荐】</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/11/11/Maven-dependencies-dependencyManagement/">Maven 中 dependencies 与 dependencyManagement 的区别</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/10/28/Data-Desensitization/">小白谈数据脱敏</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/10/21/HBase-metrics/">HBase 集群监控</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/10/18/ElasticSearch-nodes-metrics/">Elasticsearch 系列文章（四）：ElasticSearch 单个节点监控</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/10/15/ElasticSearch-cluster-health-metrics/">Elasticsearch 系列文章（三）：ElasticSearch 集群监控</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/10/14/Nexus3-Maven/">Centos7 搭建最新 Nexus3 Maven 私服</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/10/14/JsonPath/">JsonPath —— JSON 解析神器</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/09/23/Guava-limit/">Google Guava 缓存实现接口的限流</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/09/17/Interview-summary/">面试过阿里等互联网大公司，我知道了这些套路</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/09/15/linux-lua-lfs-install/">Linux 下 lua 开发环境安装及安装 luafilesystem</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/09/09/Elasticsearch-install/">Elasticsearch 系列文章（二）：全文搜索引擎 Elasticsearch 集群搭建入门教程</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/09/08/Elasticsearch-analyzers/">Elasticsearch 系列文章（一）：Elasticsearch 默认分词器和中分分词器之间的比较及使用方法</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/28/recommend-books/">那些年我看过的书 —— 致敬我的大学生活 —— Say Good Bye ！</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/18/Ubuntu-install-Nginx/">Ubuntu16.10 安装 Nginx</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/11/most-success/">马云热血励志演讲《最伟大的成功》</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/08/android-projects/">源码大招：不服来战！撸这些完整项目，你不牛逼都难！</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/05/Nginx/">Nginx 基本知识快速入门</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/04/alibaba/">秋招第三站 —— 内推阿里（一面）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/04/yaxin/">秋招第一站 —— 亚信科技</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/08/04/iqiyi/">秋招第二站 —— 内推爱奇艺（一面二面）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/07/29/ThreadPool/">Java 线程池艺术探索</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/07/25/Java-performance-tuning/">Java 性能调优需要格外注意的细节</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/07/21/Spring-MVC03/">Spring MVC系列文章（五）：看透 Spring MVC 源代码分析与实践 ——  Spring MVC 组件分析</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/07/14/Spring-MVC01/">Spring MVC系列文章（三）：看透 Spring MVC 源代码分析与实践 ——  网站基础知识</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/07/14/Spring-MVC02/">Spring MVC系列文章（四）：看透 Spring MVC 源代码分析与实践 ——  俯视 Spring MVC</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/07/09/servlet/">通过源码详解 Servlet</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/28/Velocity-foreach/">Velocity 循环指令一种好的解决方法</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/23/java-io/">Java IO流学习超详细总结（图文并茂）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/23/AJAX/">AJAX 学习</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/20/Java-error1/">java.sql.SQLException Field 'id' doesn't have a default value</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/19/中缀表达式转换成前缀和后缀表达式这类题目的超实用解题技巧/">中缀表达式转换成前缀和后缀表达式这类题目的超实用解题技巧</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/18/Bootstrap入门需掌握的知识点（二）/">Bootstrap入门需掌握的知识点（二）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/18/Bootstrap入门需掌握的知识点（一）/">Bootstrap入门需掌握的知识点（一）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/18/循环队列的相关条件和公式/">循环队列的相关条件和公式</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/17/详解 Filter 过滤器/">详解 Filter 过滤器</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/17/blog-talk/">搭建一个博客项目后的碎碎念</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/17/详细深入分析 Java ClassLoader 工作机制/">详细深入分析 Java ClassLoader 工作机制</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/16/通过项目逐步深入了解Spring MVC（一）/">通过项目逐步深入了解Spring MVC（一）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/15/通过项目逐步深入了解Mybatis(四)/">通过项目逐步深入了解Mybatis（四）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/14/通过项目逐步深入了解Mybatis(三)/">通过项目逐步深入了解Mybatis（三）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Hexo-yilia-toc/">Hexo + yilia 主题实现文章目录</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/HashMap-Hashtable/">HashMap、Hashtable、HashSet 和 ConcurrentHashMap 的比较</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Spring MVC+Hibernate JPA搭建的博客系统项目中所遇到的坑/">Spring MVC系列文章（二）：Spring MVC+Hibernate JPA搭建的博客系统项目中所遇到的坑</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Spring MVC + Hibernate JPA + Bootstrap 搭建的博客系统/">Spring MVC系列文章（一）：Spring MVC + Hibernate JPA + Bootstrap 搭建的博客系统 Demo</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Java-Thread/">《Java 多线程编程核心技术》学习笔记及总结</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Python爬虫实战之爬取糗事百科段子/">Python爬虫实战之爬取糗事百科段子</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/通过项目逐步深入了解Mybatis(二)/">通过项目逐步深入了解Mybatis（二）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Pyspider框架 —— Python爬虫实战之爬取 V2EX 网站帖子/">Pyspider框架 —— Python爬虫实战之爬取 V2EX 网站帖子</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/MySQL-select-good/">MySQL 处理海量数据时的一些优化查询速度方法</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/MyBatis-foreach/">MyBatis的foreach语句详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Java连接Oracle数据库的三种连接方式/">Java连接Oracle数据库的三种连接方式</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/【字符串】判断两字符串是否互为旋转词？/">【字符串】判断两字符串是否互为旋转词？</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/【字符串】字符串逆序/">字符串</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/feiji/">记录下自己第一次坐飞机的感受</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/JVM性能调优监控工具jps、jstack、jmap、jhat、jstat等使用详解/">JVM性能调优监控工具jps、jstack、jmap、jhat、jstat等使用详解</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Java-16-lession/">《疯狂 Java 突破程序员基本功的 16 课》读书笔记</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/奇怪的Java题：为什么128 == 128返回为False，而127 == 127会返回为True-/">奇怪的Java题：为什么128 == 128返回为False，而127 == 127会返回为True?</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/利用Github Page 搭建个人博客网站/">利用Github Page 搭建个人博客网站</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/深入分析 Java Web 中的中文编码问题/">深入分析 Java Web 中的中文编码问题</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/程序访问文件的几种方式/">程序访问文件的几种方式</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/深度探究Java 中 finally 语句块/">深度探究Java 中 finally 语句块</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/解决jdk1.8中发送邮件失败（handshake_failure）问题/">解决jdk1.8中发送邮件失败（handshake_failure）问题</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/java-var/">从对象深入分析 Java 中实例变量和类变量的区别</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/String-new/">关于String s = new String("xyz"); 创建几个对象的问题</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/java读取文件/">Java读取文件</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Java NIO 系列教程/">Java NIO 系列教程</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/13/Python爬虫实战之爬取百度贴吧帖子/">Python爬虫实战之爬取百度贴吧帖子</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/12/通过项目逐步深入了解Mybatis(一)/">通过项目逐步深入了解Mybatis（一）</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/06/02/poetry2/"> 六月 —— 愿你做最美好的自己！</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/05/12/poetry/">最近很火的鸡汤，分享给大家</a></li><li class="post-list-item"><a class="post-list-link" href="/2017/04/13/Hexo-yilia-changyan/">Github pages + Hexo 博客 yilia 主题使用畅言评论系统</a></li></ul>
    <script src="https://7.url.cn/edu/jslib/comb/require-2.1.6,jquery-1.9.1.min.js"></script>
    <script>
        $(".post-list").addClass("toc-article");
        $(".post-list-item a").attr("target","_blank");
        $("#post-nav-button > a:nth-child(2)").click(function() {
            $(".fa-bars, .fa-times").toggle();
            $(".post-list").toggle(300);
            if ($(".toc").length > 0) {
                $("#toc, #tocButton").toggle(200, function() {
                    if ($(".switch-area").is(":visible")) {
                        $("#tocButton").attr("value", valueHide);
                        }
                    })
            }
            else {
            }
        })
    </script>



    <script>
        
    </script>
</div>
      <footer id="footer">
    <div class="outer">
        <div id="footer-info">
            <div class="footer-left">
                &copy; 2019 zhisheng
            </div>
        </div>
        
            <div class="visit">
                
                    <span id="busuanzi_container_site_pv" style='display:none'>
                        <span id="site-visit" >本站总访问量: 
                            <span id="busuanzi_value_site_pv"></span>
                        </span>
                    </span>
                
                
                    <span>, </span>
                
                
                    <span id="busuanzi_container_page_pv" style='display:none'>
                        <span id="page-visit">本文总阅读量: 
                            <span id="busuanzi_value_page_pv"></span>
                        </span>
                    </span>
                
            </div>
        
    </div>
</footer>

    </div>
    <script src="https://7.url.cn/edu/jslib/comb/require-2.1.6,jquery-1.9.1.min.js"></script>
<script src="/js/main.js"></script>

    <script>
        $(document).ready(function() {
            var backgroundnum = 24;
            var backgroundimg = "url(/background/bg-x.jpg)".replace(/x/gi, Math.ceil(Math.random() * backgroundnum));
            $("#mobile-nav").css({"background-image": backgroundimg,"background-size": "cover","background-position": "center"});
            $(".left-col").css({"background-image": backgroundimg,"background-size": "cover","background-position": "center"});
        })
    </script>




	<script>
	var _hmt = _hmt || [];
	(function() {
	  var hm = document.createElement("script");
	  hm.src = "https://hm.baidu.com/hm.js?c41be5a0c9f014e977695f66c065b5d3";
	  var s = document.getElementsByTagName("script")[0]; 
	  s.parentNode.insertBefore(hm, s);
	})();
	</script>


<div class="scroll" id="scroll">
    <a href="#"><i class="fa fa-arrow-up"></i></a>
    <a href="#comments"><i class="fa fa-comments-o"></i></a>
    <a href="#footer"><i class="fa fa-arrow-down"></i></a>
</div>
<script>
    $(document).ready(function() {
        if ($("#comments").length < 1) {
            $("#scroll > a:nth-child(2)").hide();
        };
    })
</script>

<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js">
</script>

  <script language="javascript">
    $(function() {
        $("a[title]").each(function() {
            var a = $(this);
            var title = a.attr('title');
            if (title == undefined || title == "") return;
            a.data('title', title).removeAttr('title').hover(

            function() {
                var offset = a.offset();
                $("<div id=\"anchortitlecontainer\"></div>").appendTo($("body")).html(title).css({
                    top: offset.top - a.outerHeight() - 15,
                    left: offset.left + a.outerWidth()/2 + 1
                }).fadeIn(function() {
                    var pop = $(this);
                    setTimeout(function() {
                        pop.remove();
                    }, pop.text().length * 800);
                });
            }, function() {
                $("#anchortitlecontainer").remove();
            });
        });
    });
</script>


  </div>
</body>
</html>